Console [eagle-46vm1] log opened at 2014-10-26 03:45:03 UTC. LNet: Service thread pid 10891 was inactive for 40.00s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: Pid: 10891, comm: ll_ost_io00_016 Call Trace: [] ? ktime_get_ts+0xb1/0xf0 [] io_schedule+0x73/0xc0 [] cv_wait_common+0x8c/0x100 [spl] [] ? autoremove_wake_function+0x0/0x40 [] __cv_wait_io+0x18/0x20 [spl] [] zio_wait+0xfb/0x1b0 [zfs] [] dmu_buf_hold_array_by_dnode+0x19d/0x4c0 [zfs] [] dmu_buf_hold_array_by_bonus+0x68/0x90 [zfs] [] osd_bufs_get+0x493/0xb00 [osd_zfs] [] ? lu_object_find+0x16/0x20 [obdclass] [] ofd_preprw_read+0x14b/0x7f0 [ofd] [] ofd_preprw+0x77a/0x1480 [ofd] [] ? lustre_pack_reply_v2+0x1e1/0x280 [ptlrpc] [] obd_preprw+0x121/0x390 [ost] [] ost_brw_read+0xd29/0x1350 [ost] [] ? vsnprintf+0x336/0x5e0 [] ? cfs_set_ptldebug_header+0x2b/0xc0 [libcfs] [] ? lustre_msg_get_version+0x8c/0x100 [ptlrpc] [] ? lustre_msg_check_version+0xe8/0x100 [ptlrpc] [] ost_handle+0x24a8/0x44d0 [ost] [] ? libcfs_id2str+0x74/0xb0 [libcfs] [] ptlrpc_server_handle_request+0x385/0xc00 [ptlrpc] [] ? lc_watchdog_touch+0x65/0x170 [libcfs] [] ? ptlrpc_wait_event+0xa9/0x2d0 [ptlrpc] [] ? default_wake_function+0x0/0x20 [] ptlrpc_main+0xaed/0x1770 [ptlrpc] [] ? ptlrpc_main+0x0/0x1770 [ptlrpc] [] kthread+0x96/0xa0 [] child_rip+0xa/0x20 [] ? kthread+0x0/0xa0 [] ? child_rip+0x0/0x20 LustreError: dumping log to /tmp/lustre-log.1414295114.10891 Pid: 10887, comm: ll_ost_io00_012 Call Trace: [] ? ktime_get_ts+0xb1/0xf0 [] io_schedule+0x73/0xc0 [] cv_wait_common+0x8c/0x100 [spl] [] ? autoremove_wake_function+0x0/0x40 [] __cv_wait_io+0x18/0x20 [spl] [] zio_wait+0xfb/0x1b0 [zfs] [] dmu_buf_hold_array_by_dnode+0x19d/0x4c0 [zfs] [] dmu_buf_hold_array_by_bonus+0x68/0x90 [zfs] [] osd_bufs_get+0x493/0xb00 [osd_zfs] [] ? number+0x2ee/0x320 [] ? lu_object_find+0x16/0x20 [obdclass] [] ofd_preprw_read+0x14b/0x7f0 [ofd] [] ofd_preprw+0x77a/0x1480 [ofd] [] ? lustre_pack_reply_v2+0x1e1/0x280 [ptlrpc] [] obd_preprw+0x121/0x390 [ost] [] ost_brw_read+0xd29/0x1350 [ost] [] ? vsnprintf+0x336/0x5e0 [] ? cfs_set_ptldebug_header+0x2b/0xc0 [libcfs] [] ? lustre_msg_get_version+0x8c/0x100 [ptlrpc] [] ? lustre_msg_check_version+0xe8/0x100 [ptlrpc] [] ost_handle+0x24a8/0x44d0 [ost] [] ? libcfs_id2str+0x74/0xb0 [libcfs] [] ptlrpc_server_handle_request+0x385/0xc00 [ptlrpc] [] ? lc_watchdog_touch+0x65/0x170 [libcfs] [] ? ptlrpc_wait_event+0xa9/0x2d0 [ptlrpc] [] ptlrpc_main+0xaed/0x1770 [ptlrpc] [] ? ptlrpc_main+0x0/0x1770 [ptlrpc] [] kthread+0x96/0xa0 [] child_rip+0xa/0x20 [] ? kthread+0x0/0xa0 [] ? child_rip+0x0/0x20 Pid: 10873, comm: ll_ost_io00_007 Call Trace: [] ? ktime_get_ts+0xb1/0xf0 [] io_schedule+0x73/0xc0 [] cv_wait_common+0x8c/0x100 [spl] [] ? autoremove_wake_function+0x0/0x40 [] __cv_wait_io+0x18/0x20 [spl] [] zio_wait+0xfb/0x1b0 [zfs] [] dmu_buf_hold_array_by_dnode+0x19d/0x4c0 [zfs] [] dmu_buf_hold_array_by_bonus+0x68/0x90 [zfs] [] osd_bufs_get+0x493/0xb00 [osd_zfs] [] ? lu_object_find+0x16/0x20 [obdclass] [] ofd_preprw_read+0x14b/0x7f0 [ofd] [] ofd_preprw+0x77a/0x1480 [ofd] [] ? lustre_pack_reply_v2+0x1e1/0x280 [ptlrpc] [] obd_preprw+0x121/0x390 [ost] [] ost_brw_read+0xd29/0x1350 [ost] [] ? vsnprintf+0x336/0x5e0 [] ? cfs_set_ptldebug_header+0x2b/0xc0 [libcfs] [] ? lustre_msg_get_version+0x8c/0x100 [ptlrpc] [] ? lustre_msg_check_version+0xe8/0x100 [ptlrpc] [] ost_handle+0x24a8/0x44d0 [ost] [] ? libcfs_id2str+0x74/0xb0 [libcfs] [] ptlrpc_server_handle_request+0x385/0xc00 [ptlrpc] [] ? lc_watchdog_touch+0x65/0x170 [libcfs] [] ? ptlrpc_wait_event+0xa9/0x2d0 [ptlrpc] [] ? default_wake_function+0x0/0x20 [] ptlrpc_main+0xaed/0x1770 [ptlrpc] [] ? ptlrpc_main+0x0/0x1770 [ptlrpc] [] kthread+0x96/0xa0 [] child_rip+0xa/0x20 [] ? kthread+0x0/0xa0 [] ? child_rip+0x0/0x20 Pid: 11263, comm: ll_ost_io00_022 Call Trace: [] ? ktime_get_ts+0xb1/0xf0 [] io_schedule+0x73/0xc0 [] cv_wait_common+0x8c/0x100 [spl] [] ? autoremove_wake_function+0x0/0x40 [] __cv_wait_io+0x18/0x20 [spl] [] zio_wait+0xfb/0x1b0 [zfs] [] dmu_buf_hold_array_by_dnode+0x19d/0x4c0 [zfs] [] dmu_buf_hold_array_by_bonus+0x68/0x90 [zfs] [] osd_bufs_get+0x493/0xb00 [osd_zfs] [] ? lu_object_find+0x16/0x20 [obdclass] [] ofd_preprw_read+0x14b/0x7f0 [ofd] [] ofd_preprw+0x77a/0x1480 [ofd] [] ? lustre_pack_reply_v2+0x1e1/0x280 [ptlrpc] [] obd_preprw+0x121/0x390 [ost] [] ost_brw_read+0xd29/0x1350 [ost] [] ? vsnprintf+0x336/0x5e0 [] ? cfs_set_ptldebug_header+0x2b/0xc0 [libcfs] [] ? lustre_msg_get_version+0x8c/0x100 [ptlrpc] [] ? lustre_msg_check_version+0xe8/0x100 [ptlrpc] [] ost_handle+0x24a8/0x44d0 [ost] [] ? libcfs_id2str+0x74/0xb0 [libcfs] [] ptlrpc_server_handle_request+0x385/0xc00 [ptlrpc] [] ? lc_watchdog_touch+0x65/0x170 [libcfs] [] ? ptlrpc_wait_event+0xa9/0x2d0 [ptlrpc] [] ? default_wake_function+0x0/0x20 [] ptlrpc_main+0xaed/0x1770 [ptlrpc] [] ? ptlrpc_main+0x0/0x1770 [ptlrpc] [] kthread+0x96/0xa0 [] child_rip+0xa/0x20 [] ? kthread+0x0/0xa0 [] ? child_rip+0x0/0x20 Pid: 10888, comm: ll_ost_io00_013 Call Trace: [] ? ktime_get_ts+0xb1/0xf0 [] io_schedule+0x73/0xc0 [] cv_wait_common+0x8c/0x100 [spl] [] ? autoremove_wake_function+0x0/0x40 [] __cv_wait_io+0x18/0x20 [spl] [] zio_wait+0xfb/0x1b0 [zfs] [] dmu_buf_hold_array_by_dnode+0x19d/0x4c0 [zfs] [] dmu_buf_hold_array_by_bonus+0x68/0x90 [zfs] [] osd_bufs_get+0x493/0xb00 [osd_zfs] [] ? lu_object_find+0x16/0x20 [obdclass] [] ofd_preprw_read+0x14b/0x7f0 [ofd] [] ofd_preprw+0x77a/0x1480 [ofd] [] ? lustre_pack_reply_v2+0x1e1/0x280 [ptlrpc] [] obd_preprw+0x121/0x390 [ost] [] ost_brw_read+0xd29/0x1350 [ost] [] ? vsnprintf+0x336/0x5e0 [] ? cfs_set_ptldebug_header+0x2b/0xc0 [libcfs] [] ? lustre_msg_get_version+0x8c/0x100 [ptlrpc] [] ? lustre_msg_check_version+0xe8/0x100 [ptlrpc] [] ost_handle+0x24a8/0x44d0 [ost] [] ? libcfs_id2str+0x74/0xb0 [libcfs] [] ptlrpc_server_handle_request+0x385/0xc00 [ptlrpc] [] ? lc_watchdog_touch+0x65/0x170 [libcfs] [] ? ptlrpc_wait_event+0xa9/0x2d0 [ptlrpc] [] ? default_wake_function+0x0/0x20 [] ptlrpc_main+0xaed/0x1770 [ptlrpc] [] ? ptlrpc_main+0x0/0x1770 [ptlrpc] [] kthread+0x96/0xa0 [] child_rip+0xa/0x20 [] ? kthread+0x0/0xa0 [] ? child_rip+0x0/0x20 LNet: Service thread pid 10892 was inactive for 40.04s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. LNet: Service thread pid 10892 completed after 43.16s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). LNet: Service thread pid 10888 completed after 45.82s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity-benchmark test pios_fpp: pios file per process == 20:50:50 $1414295450$ Lustre: DEBUG MARKER: == sanity-benchmark test pios_fpp: pios file per process == 20:50:50 (1414295450) Lustre: DEBUG MARKER: /usr/sbin/lctl mark \/usr\/bin\/pios -t 1,8,40 -n 1024 -c 1M -s 8M -o 16M -L fpp -p \/mnt\/lustre\/dpios_fpp.sanity-benchmark Lustre: DEBUG MARKER: /usr/bin/pios -t 1,8,40 -n 1024 -c 1M -s 8M -o 16M -L fpp -p /mnt/lustre/dpios_fpp.sanity-benchmark LNet: Service thread pid 10999 was inactive for 40.00s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: LNet: Skipped 4 previous similar messages Pid: 10999, comm: ll_ost00_023 Call Trace: [] ? cfs_hash_bd_get+0x25/0x80 [libcfs] [] ? nvlist_lookup_common+0x84/0xd0 [znvpair] [] ? prepare_to_wait_exclusive+0x4e/0x80 [] cv_wait_common+0xed/0x100 [spl] [] ? autoremove_wake_function+0x0/0x40 [] ? up+0x2f/0x50 [] __cv_wait+0x15/0x20 [spl] [] txg_wait_synced+0x7b/0xa0 [zfs] [] osd_object_sync+0x179/0x1c0 [osd_zfs] [] ofd_sync+0x36b/0x680 [ofd] [] ? kmem_cache_alloc_trace+0x1a3/0x1b0 [] ost_blocking_ast+0x5f0/0x1020 [ost] [] ldlm_cancel_callback+0x6c/0x1a0 [ptlrpc] [] ldlm_lock_cancel+0x5a/0x1e0 [ptlrpc] [] ldlm_handle_ast_error+0x1ee/0x550 [ptlrpc] [] ? after_reply+0x7f6/0xdb0 [ptlrpc] [] ldlm_cb_interpret+0x1ce/0x2b0 [ptlrpc] [] ptlrpc_check_set+0x2c1/0x1b50 [ptlrpc] [] ptlrpc_set_wait+0x30a/0x860 [ptlrpc] [] ? default_wake_function+0x0/0x20 [] ? ldlm_work_bl_ast_lock+0x0/0x290 [ptlrpc] [] ldlm_run_ast_work+0x1bb/0x470 [ptlrpc] [] ldlm_process_extent_lock+0x155/0xae0 [ptlrpc] [] ldlm_lock_enqueue+0x463/0x980 [ptlrpc] [] ldlm_handle_enqueue0+0x51b/0x10c0 [ptlrpc] [] ldlm_handle_enqueue+0x5e/0x70 [ptlrpc] [] ? ldlm_server_completion_ast+0x0/0x770 [ptlrpc] [] ? ost_blocking_ast+0x0/0x1020 [ost] [] ? ldlm_server_glimpse_ast+0x0/0x3b0 [ptlrpc] [] ost_handle+0x1d1d/0x44d0 [ost] [] ? libcfs_id2str+0x74/0xb0 [libcfs] [] ptlrpc_server_handle_request+0x385/0xc00 [ptlrpc] [] ? lc_watchdog_touch+0x65/0x170 [libcfs] [] ? ptlrpc_wait_event+0xa9/0x2d0 [ptlrpc] [] ptlrpc_main+0xaed/0x1770 [ptlrpc] [] ? ptlrpc_main+0x0/0x1770 [ptlrpc] [] kthread+0x96/0xa0 [] child_rip+0xa/0x20 [] ? kthread+0x0/0xa0 [] ? child_rip+0x0/0x20 LustreError: dumping log to /tmp/lustre-log.1414296399.10999 LNet: Service thread pid 10999 completed after 89.51s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). LNet: Skipped 4 previous similar messages Lustre: lustre-OST0001: Slow creates, 128/512 objects created at a rate of 2/s Lustre: lustre-OST0000: Slow creates, 128/512 objects created at a rate of 2/s Lustre: DEBUG MARKER: /usr/sbin/lctl mark \/usr\/bin\/pios -t 1,8,40 -n 1024 -c 1M -s 8M -o 16M -L fpp --verify -p \/mnt\/lustre\/dpios_fpp.sanity-benchmark Lustre: DEBUG MARKER: /usr/bin/pios -t 1,8,40 -n 1024 -c 1M -s 8M -o 16M -L fpp --verify -p /mnt/lustre/dpios_fpp.sanity-benchmark spl_kmem_cache/ invoked oom-killer: gfp_mask=0x84d0, order=0, oom_adj=0, oom_score_adj=0 spl_kmem_cache/ cpuset=/ mems_allowed=0 Pid: 395, comm: spl_kmem_cache/ Tainted: P --------------- 2.6.32-431.29.2.el6_lustre.g9835a2a.x86_64 #1 Call Trace: [] ? cpuset_print_task_mems_allowed+0x91/0xb0 [] ? dump_header+0x90/0x1b0 [] ? check_panic_on_oom+0x4e/0x80 [] ? out_of_memory+0x1bb/0x3c0 [] ? zone_statistics+0x99/0xc0 [] ? __alloc_pages_nodemask+0x89f/0x8d0 [] ? alloc_pages_current+0xaa/0x110 [] ? __get_free_pages+0xe/0x50 [] ? pte_alloc_one_kernel+0x15/0x20 [] ? __pte_alloc_kernel+0x1b/0xc0 [] ? vmap_page_range_noflush+0x309/0x370 [] ? map_vm_area+0x32/0x50 [] ? __vmalloc_area_node+0x100/0x190 [] ? kv_alloc+0x59/0x60 [spl] [] ? __vmalloc_node+0xad/0x120 [] ? kv_alloc+0x59/0x60 [spl] [] ? __vmalloc+0x22/0x30 [] ? kv_alloc+0x59/0x60 [spl] [] ? spl_cache_grow_work+0x39/0x2d0 [spl] [] ? __wake_up+0x53/0x70 [] ? taskq_thread+0x1e7/0x3f0 [spl] [] ? default_wake_function+0x0/0x20 [] ? taskq_thread+0x0/0x3f0 [spl] [] ? kthread+0x96/0xa0 [] ? child_rip+0xa/0x20 [] ? kthread+0x0/0xa0 [] ? child_rip+0x0/0x20 Mem-Info: Node 0 DMA per-cpu: CPU 0: hi: 0, btch: 1 usd: 0 Node 0 DMA32 per-cpu: CPU 0: hi: 186, btch: 31 usd: 17 active_anon:0 inactive_anon:11 isolated_anon:0 active_file:107 inactive_file:109 isolated_file:0 unevictable:0 dirty:0 writeback:16 unstable:0 free:12598 slab_reclaimable:1537 slab_unreclaimable:12274 mapped:91 shmem:0 pagetables:1322 bounce:0 Node 0 DMA free:8184kB min:332kB low:412kB high:496kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15348kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:4kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes lowmem_reserve[]: 0 2004 2004 2004 Node 0 DMA32 free:42208kB min:44720kB low:55900kB high:67080kB active_anon:0kB inactive_anon:44kB active_file:428kB inactive_file:436kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:2052308kB mlocked:0kB dirty:0kB writeback:64kB mapped:364kB shmem:0kB slab_reclaimable:6148kB slab_unreclaimable:49092kB kernel_stack:3472kB pagetables:5288kB unstable:0kB bounce:0kB writeback_tmp:0kB pages_scanned:184 all_unreclaimable? no lowmem_reserve[]: 0 0 0 0 Node 0 DMA: 0*4kB 1*8kB 1*16kB 1*32kB 1*64kB 1*128kB 1*256kB 1*512kB 1*1024kB 1*2048kB 1*4096kB = 8184kB Node 0 DMA32: 578*4kB 327*8kB 212*16kB 115*32kB 52*64kB 26*128kB 18*256kB 13*512kB 4*1024kB 2*2048kB 1*4096kB = 42208kB 250 total pagecache pages 14 pages in swap cache Swap cache stats: add 6015, delete 6001, find 196/324 Free swap = 4108320kB Total swap = 4128764kB 524284 pages RAM 43654 pages reserved 1393 pages shared 461098 pages non-shared [ pid ] uid tgid total_vm rss cpu oom_adj oom_score_adj name [ 351] 0 351 2732 12 0 -17 -1000 udevd [ 680] 0 680 2731 11 0 -17 -1000 udevd [ 1046] 0 1046 2280 17 0 0 0 dhclient [ 1098] 0 1098 6910 29 0 -17 -1000 auditd [ 1114] 0 1114 63854 28 0 0 0 rsyslogd [ 1154] 32 1154 4744 21 0 0 0 rpcbind [ 1166] 0 1166 49886 78 0 0 0 sssd [ 1167] 0 1167 64220 75 0 0 0 sssd_be [ 1168] 0 1168 50377 70 0 0 0 sssd_nss [ 1169] 0 1169 47954 70 0 0 0 sssd_pam [ 1170] 0 1170 47403 70 0 0 0 sssd_ssh [ 1171] 0 1171 52533 71 0 0 0 sssd_pac [ 1188] 29 1188 6357 28 0 0 0 rpc.statd [ 6805] 81 6805 5871 18 0 0 0 dbus-daemon [ 6843] 0 6843 1020 16 0 0 0 acpid [ 6852] 68 6852 9927 61 0 0 0 hald [ 6853] 0 6853 5082 55 0 0 0 hald-runner [ 6885] 0 6885 5612 55 0 0 0 hald-addon-inpu [ 6902] 68 6902 4484 56 0 0 0 hald-addon-acpi [ 6916] 0 6916 151443 28 0 0 0 automount [ 6957] 0 6957 26827 5 0 0 0 rpc.rquotad [ 6961] 0 6961 5414 18 0 0 0 rpc.mountd [ 6997] 0 6997 5773 15 0 0 0 rpc.idmapd [ 7028] 496 7028 56787 28 0 0 0 munged [ 7043] 0 7043 16656 17 0 -17 -1000 sshd [ 7051] 0 7051 5545 18 0 0 0 xinetd [ 7059] 38 7059 8205 27 0 0 0 ntpd [ 7135] 0 7135 20846 26 0 0 0 master [ 7145] 89 7145 20909 24 0 0 0 qmgr [ 7158] 0 7158 29324 21 0 0 0 crond [ 7169] 0 7169 5385 10 0 0 0 atd [ 7197] 0 7197 15590 57 0 0 0 certmonger [ 7209] 0 7209 1016 14 0 0 0 mingetty [ 7211] 0 7211 1020 15 0 0 0 agetty [ 7212] 0 7212 1016 14 0 0 0 mingetty [ 7214] 0 7214 1016 14 0 0 0 mingetty [ 7216] 0 7216 1016 14 0 0 0 mingetty [ 7218] 0 7218 1016 14 0 0 0 mingetty [ 7220] 0 7220 1016 14 0 0 0 mingetty [10941] 0 10941 26134 26 0 0 0 sshd [10943] 0 10943 27085 15 0 0 0 bash [13395] 89 13395 20866 24 0 0 0 pickup Kernel panic - not syncing: Out of memory: system-wide panic_on_oom is enabled Pid: 395, comm: spl_kmem_cache/ Tainted: P --------------- 2.6.32-431.29.2.el6_lustre.g9835a2a.x86_64 #1 Call Trace: [] ? panic+0xa7/0x16f [] ? dump_header+0x191/0x1b0 [] ? check_panic_on_oom+0x7c/0x80 [] ? out_of_memory+0x1bb/0x3c0 [] ? zone_statistics+0x99/0xc0 [] ? __alloc_pages_nodemask+0x89f/0x8d0 [] ? alloc_pages_current+0xaa/0x110 [] ? __get_free_pages+0xe/0x50 [] ? pte_alloc_one_kernel+0x15/0x20 [] ? __pte_alloc_kernel+0x1b/0xc0 [] ? vmap_page_range_noflush+0x309/0x370 [] ? map_vm_area+0x32/0x50 [] ? __vmalloc_area_node+0x100/0x190 [] ? kv_alloc+0x59/0x60 [spl] [] ? __vmalloc_node+0xad/0x120 [] ? kv_alloc+0x59/0x60 [spl] [] ? __vmalloc+0x22/0x30 [] ? kv_alloc+0x59/0x60 [spl] [] ? spl_cache_grow_work+0x39/0x2d0 [spl] [] ? __wake_up+0x53/0x70 [] ? taskq_thread+0x1e7/0x3f0 [spl] [] ? default_wake_function+0x0/0x20 [] ? taskq_thread+0x0/0x3f0 [spl] [] ? kthread+0x96/0xa0 [] ? child_rip+0xa/0x20 [] ? kthread+0x0/0xa0 [] ? child_rip+0x0/0x20 Console [eagle-46vm1] joined by on pts/1 at 10-27 04:43. Console [eagle-46vm1] departed by on pts/1 at 10-27 04:48. Press any key to continue. [H[J GNU GRUB version 0.97 (631K lower / 2096116K upper memory) [m[4;2H+-------------------------------------------------------------------------+[5;2H|[5;76H|[6;2H|[6;76H|[7;2H|[7;76H|[8;2H|[8;76H|[9;2H|[9;76H|[10;2H|[10;76H|[11;2H|[11;76H|[12;2H|[12;76H|[13;2H|[13;76H|[14;2H|[14;76H|[15;2H|[15;76H|[16;2H|[16;76H|[17;2H+-------------------------------------------------------------------------+[m Use the ^ and v keys to select which entry is highlighted. Press enter to boot the selected OS, 'e' to edit the commands before booting, 'a' to modify the kernel arguments before booting, or 'c' for a command-line.[5;78H [m[7m[5;3H CentOS (2.6.32-431.29.2.el6_lustre.g9835a2a.x86_64) [5;75H[m[m[6;3H [6;75H[m[m[7;3H [7;75H[m[m[8;3H [8;75H[m[m[9;3H [9;75H[m[m[10;3H [10;75H[m[m[11;3H [11;75H[m[m[12;3H [12;75H[m[m[13;3H [13;75H[m[m[14;3H [14;75H[m[m[15;3H [15;75H[m[m[16;3H [16;75H[m[16;78H [5;75H[23;4HThe highlighted entry will be booted automatically in 5 seconds. [5;75H[23;4H [5;75H[H[JInitializing cgroup subsys cpuset Initializing cgroup subsys cpu Linux version 2.6.32-431.29.2.el6_lustre.g9835a2a.x86_64 (jenkins@onyx-8-sde1-el6-x8664.onyx.hpdd.intel.com) (gcc version 4.4.7 20120313 (Red Hat 4.4.7-11) (GCC) ) #1 SMP Tue Oct 7 07:21:59 PDT 2014 Command line: ro root=UUID=21c1d2ba-fd11-46fc-a5c5-c5e8128b4cba rd_NO_LUKS rd_NO_LVM LANG=en_US.UTF-8 rd_NO_MD console=tty0 crashkernel=auto SYSFONT=latarcyrheb-sun16 KEYBOARDTYPE=pc KEYTABLE=us rd_NO_DM console=ttyS0,115200 KERNEL supported cpus: Intel GenuineIntel AMD AuthenticAMD Centaur CentaurHauls BIOS-provided physical RAM map: BIOS-e820: 0000000000000000 - 000000000009dc00 (usable) BIOS-e820: 000000000009dc00 - 00000000000a0000 (reserved) BIOS-e820: 00000000000f0000 - 0000000000100000 (reserved) BIOS-e820: 0000000000100000 - 000000007fffd000 (usable) BIOS-e820: 000000007fffd000 - 0000000080000000 (reserved) BIOS-e820: 00000000fffbc000 - 0000000100000000 (reserved) DMI 2.4 present. SMBIOS version 2.4 @ 0xFDA30 Hypervisor detected: KVM last_pfn = 0x7fffd max_arch_pfn = 0x400000000 x86 PAT enabled: cpu 0, old 0x70406, new 0x7010600070106 init_memory_mapping: 0000000000000000-000000007fffd000 RAMDISK: 36aa5000 - 37fefc7e ACPI: RSDP 00000000000fda00 00014 (v00 BOCHS ) ACPI: RSDT 000000007fffd670 00034 (v01 BOCHS BXPCRSDT 00000001 BXPC 00000001) ACPI: FACP 000000007ffffe40 00074 (v01 BOCHS BXPCFACP 00000001 BXPC 00000001) ACPI: DSDT 000000007fffd950 024A2 (v01 BXPC BXDSDT 00000001 INTL 20090123) ACPI: FACS 000000007ffffe00 00040 ACPI: SSDT 000000007fffd8b0 0009E (v01 BOCHS BXPCSSDT 00000001 BXPC 00000001) ACPI: APIC 000000007fffd7c0 00078 (v01 BOCHS BXPCAPIC 00000001 BXPC 00000001) ACPI: SSDT 000000007fffd6b0 0010F (v01 BXPC BXSSDTPC 00000001 INTL 20090123) Setting APIC routing to flat. No NUMA configuration found Faking a node at 0000000000000000-000000007fffd000 Bootmem setup node 0 0000000000000000-000000007fffd000 NODE_DATA [000000000000a000 - 000000000003dfff] bootmap [000000000003e000 - 000000000004dfff] pages 10 (7 early reservations) ==> bootmem [0000000000 - 007fffd000] #0 [0000000000 - 0000001000] BIOS data page ==> [0000000000 - 0000001000] #1 [0000006000 - 0000008000] TRAMPOLINE ==> [0000006000 - 0000008000] #2 [0001000000 - 0002020aa4] TEXT DATA BSS ==> [0001000000 - 0002020aa4] #3 [0036aa5000 - 0037fefc7e] RAMDISK ==> [0036aa5000 - 0037fefc7e] #4 [000009dc00 - 0000100000] BIOS reserved ==> [000009dc00 - 0000100000] #5 [0002021000 - 0002021081] BRK ==> [0002021000 - 0002021081] #6 [0000008000 - 000000a000] PGTABLE ==> [0000008000 - 000000a000] found SMP MP-table at [ffff8800000fda50] fda50 Reserving 129MB of memory at 48MB for crashkernel (System RAM: 2047MB) kvm-clock: Using msrs 4b564d01 and 4b564d00 kvm-clock: cpu 0, msr 0:1c24801, boot clock Zone PFN ranges: DMA 0x00000001 -> 0x00001000 DMA32 0x00001000 -> 0x00100000 Normal 0x00100000 -> 0x00100000 Movable zone start PFN for each node early_node_map[2] active PFN ranges 0: 0x00000001 -> 0x0000009d 0: 0x00000100 -> 0x0007fffd ACPI: PM-Timer IO Port: 0xb008 Setting APIC routing to flat. ACPI: LAPIC (acpi_id[0x00] lapic_id[0x00] enabled) ACPI: LAPIC_NMI (acpi_id[0xff] dfl dfl lint[0x1]) ACPI: IOAPIC (id[0x00] address[0xfec00000] gsi_base[0]) IOAPIC[0]: apic_id 0, version 17, address 0xfec00000, GSI 0-23 ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl) ACPI: INT_SRC_OVR (bus 0 bus_irq 5 global_irq 5 high level) ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level) ACPI: INT_SRC_OVR (bus 0 bus_irq 10 global_irq 10 high level) ACPI: INT_SRC_OVR (bus 0 bus_irq 11 global_irq 11 high level) Using ACPI (MADT) for SMP configuration information SMP: Allowing 1 CPUs, 0 hotplug CPUs PM: Registered nosave memory: 000000000009d000 - 000000000009e000 PM: Registered nosave memory: 000000000009e000 - 00000000000a0000 PM: Registered nosave memory: 00000000000a0000 - 00000000000f0000 PM: Registered nosave memory: 00000000000f0000 - 0000000000100000 Allocating PCI resources starting at 80000000 (gap: 80000000:7ffbc000) Booting paravirtualized kernel on KVM NR_CPUS:4096 nr_cpumask_bits:1 nr_cpu_ids:1 nr_node_ids:1 PERCPU: Embedded 31 pages/cpu @ffff880002200000 s94936 r8192 d23848 u2097152 pcpu-alloc: s94936 r8192 d23848 u2097152 alloc=1*2097152 pcpu-alloc: [0] 0 kvm-clock: cpu 0, msr 0:2216801, primary cpu clock kvm-stealtime: cpu 0, msr 220e880 Built 1 zonelists in Node order, mobility grouping on. Total pages: 516914 Policy zone: DMA32 Kernel command line: ro root=UUID=21c1d2ba-fd11-46fc-a5c5-c5e8128b4cba rd_NO_LUKS rd_NO_LVM LANG=en_US.UTF-8 rd_NO_MD console=tty0 crashkernel=129M@0M SYSFONT=latarcyrheb-sun16 KEYBOARDTYPE=pc KEYTABLE=us rd_NO_DM console=ttyS0,115200 PID hash table entries: 4096 (order: 3, 32768 bytes) Checking aperture... No AGP bridge found AMD-Vi disabled by default: pass amd_iommu=on to enable Memory: 1897020k/2097140k available (5332k kernel code, 400k absent, 199720k reserved, 7006k data, 1280k init) Hierarchical RCU implementation. NR_IRQS:33024 nr_irqs:256 Console: colour VGA+ 80x25 console [tty0] enabled console [ttyS0] enabled allocated 8388608 bytes of page_cgroup please try 'cgroup_disable=memory' option if you don't want memory cgroups Detected 2000.138 MHz processor. Calibrating delay loop (skipped) preset value.. 4000.27 BogoMIPS (lpj=2000138) pid_max: default: 32768 minimum: 301 Security Framework initialized SELinux: Initializing. Dentry cache hash table entries: 262144 (order: 9, 2097152 bytes) Inode-cache hash table entries: 131072 (order: 8, 1048576 bytes) Mount-cache hash table entries: 256 Initializing cgroup subsys ns Initializing cgroup subsys cpuacct Initializing cgroup subsys memory Initializing cgroup subsys devices Initializing cgroup subsys freezer Initializing cgroup subsys net_cls Initializing cgroup subsys blkio Initializing cgroup subsys perf_event Initializing cgroup subsys net_prio mce: CPU supports 10 MCE banks alternatives: switching to unfair spinlock SMP alternatives: switching to UP code Freeing SMP alternatives: 36k freed ACPI: Core revision 20090903 ftrace: converting mcount calls to 0f 1f 44 00 00 ftrace: allocating 21794 entries in 86 pages APIC routing finalized to flat. ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1 CPU0: AMD QEMU Virtual CPU version (cpu64-rhel6) stepping 03 Performance Events: Broken PMU hardware detected, using software events only. NMI watchdog disabled (cpu0): hardware events not enabled Brought up 1 CPUs Total of 1 processors activated (4000.27 BogoMIPS). devtmpfs: initialized regulator: core version 0.5 NET: Registered protocol family 16 ACPI: bus type pci registered PCI: Using configuration type 1 for base access bio: create slab at 0 ACPI: Interpreter enabled ACPI: (supports S0 S5) ACPI: Using IOAPIC for interrupt routing ACPI: No dock devices found. PCI: Ignoring host bridge windows from ACPI; if necessary, use "pci=use_crs" and report a bug ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-ff]) PCI host bridge to bus 0000:00 pci_bus 0000:00: root bus resource [io 0x0000-0xffff] pci_bus 0000:00: root bus resource [mem 0x00000000-0xffffffffffff] pci 0000:00:01.3: quirk: [io 0xb000-0xb03f] claimed by PIIX4 ACPI pci 0000:00:01.3: quirk: [io 0xb100-0xb10f] claimed by PIIX4 SMB ACPI: PCI Interrupt Link [LNKA] (IRQs 5 *10 11) ACPI: PCI Interrupt Link [LNKB] (IRQs 5 *10 11) ACPI: PCI Interrupt Link [LNKC] (IRQs 5 10 *11) ACPI: PCI Interrupt Link [LNKD] (IRQs 5 10 *11) ACPI: PCI Interrupt Link [LNKS] (IRQs *9) vgaarb: device added: PCI:0000:00:02.0,decodes=io+mem,owns=io+mem,locks=none vgaarb: loaded vgaarb: bridge control possible 0000:00:02.0 SCSI subsystem initialized usbcore: registered new interface driver usbfs usbcore: registered new interface driver hub usbcore: registered new device driver usb PCI: Using ACPI for IRQ routing NetLabel: Initializing NetLabel: domain hash size = 128 NetLabel: protocols = UNLABELED CIPSOv4 NetLabel: unlabeled traffic allowed by default Switching to clocksource kvm-clock pnp: PnP ACPI init ACPI: bus type pnp registered pnp: PnP ACPI: found 6 devices ACPI: ACPI bus type pnp unregistered NET: Registered protocol family 2 IP route cache hash table entries: 65536 (order: 7, 524288 bytes) TCP established hash table entries: 262144 (order: 10, 4194304 bytes) TCP bind hash table entries: 65536 (order: 8, 1048576 bytes) TCP: Hash tables configured (established 262144 bind 65536) TCP reno registered NET: Registered protocol family 1 pci 0000:00:00.0: Limiting direct PCI/PCI transfers pci 0000:00:01.0: PIIX3: Enabling Passive Release pci 0000:00:01.0: Activating ISA DMA hang workarounds ACPI: PCI Interrupt Link [LNKD] enabled at IRQ 11 pci 0000:00:01.2: PCI INT D -> Link[LNKD] -> GSI 11 (level, high) -> IRQ 11 pci 0000:00:01.2: PCI INT D disabled Trying to unpack rootfs image as initramfs... Freeing initrd memory: 21803k freed audit: initializing netlink socket (disabled) type=2000 audit(1414385874.804:1): initialized HugeTLB registered 2 MB page size, pre-allocated 0 pages VFS: Disk quotas dquot_6.5.2 Dquot-cache hash table entries: 512 (order 0, 4096 bytes) msgmni has been set to 3747 alg: No test for stdrng (krng) ksign: Installing public key data Loading keyring - Added public key 3B887A80AE5E5B8D - User ID: Red Hat, Inc. (Kernel Module GPG key) - Added public key D4A26C9CCD09BEDA - User ID: Red Hat Enterprise Linux Driver Update Program Block layer SCSI generic (bsg) driver version 0.4 loaded (major 251) io scheduler noop registered io scheduler anticipatory registered io scheduler deadline registered (default) io scheduler cfq registered pci_hotplug: PCI Hot Plug PCI Core version: 0.5 pciehp: PCI Express Hot Plug Controller Driver version: 0.4 acpiphp: ACPI Hot Plug PCI Controller Driver version: 0.5 acpiphp: Slot [1] registered acpiphp: Slot [2] registered acpiphp: Slot [3] registered acpiphp: Slot [4] registered acpiphp: Slot [5] registered acpiphp: Slot [6] registered acpiphp: Slot [7] registered acpiphp: Slot [8] registered acpiphp: Slot [9] registered acpiphp: Slot [10] registered acpiphp: Slot [11] registered acpiphp: Slot [12] registered acpiphp: Slot [13] registered acpiphp: Slot [14] registered acpiphp: Slot [15] registered acpiphp: Slot [16] registered acpiphp: Slot [17] registered acpiphp: Slot [18] registered acpiphp: Slot [19] registered acpiphp: Slot [20] registered acpiphp: Slot [21] registered acpiphp: Slot [22] registered acpiphp: Slot [23] registered acpiphp: Slot [24] registered acpiphp: Slot [25] registered acpiphp: Slot [26] registered acpiphp: Slot [27] registered acpiphp: Slot [28] registered acpiphp: Slot [29] registered acpiphp: Slot [30] registered acpiphp: Slot [31] registered ipmi message handler version 39.2 IPMI System Interface driver. ipmi_si: Adding default-specified kcs state machine ipmi_si: Trying default-specified kcs state machine at i/o address 0xca2, slave address 0x0, irq 0 ipmi_si: Interface detection failed ipmi_si: Adding default-specified smic state machine ipmi_si: Trying default-specified smic state machine at i/o address 0xca9, slave address 0x0, irq 0 ipmi_si: Interface detection failed ipmi_si: Adding default-specified bt state machine ipmi_si: Trying default-specified bt state machine at i/o address 0xe4, slave address 0x0, irq 0 ipmi_si: Interface detection failed ipmi_si: Unable to find any System Interface(s) input: Power Button as /devices/LNXSYSTM:00/LNXPWRBN:00/input/input0 ACPI: Power Button [PWRF] [Firmware Bug]: No valid trip found GHES: HEST is not enabled! Non-volatile memory driver v1.3 Linux agpgart interface v0.103 crash memory driver: version 1.1 Serial: 8250/16550 driver, 4 ports, IRQ sharing enabled erial8250: ttyS0 at I/O 0x3f8 (irq = 4) is a 16550A 00:05: ttyS0 at I/O 0x3f8 (irq = 4) is a 16550A brd: module loaded loop: module loaded input: Macintosh mouse button emulation as /devices/virtual/input/input1 Fixed MDIO Bus: probed ehci_hcd: USB 2.0 'Enhanced' Host Controller (EHCI) Driver ohci_hcd: USB 1.1 'Open' Host Controller (OHCI) Driver uhci_hcd: USB Universal Host Controller Interface driver uhci_hcd 0000:00:01.2: PCI INT D -> Link[LNKD] -> GSI 11 (level, high) -> IRQ 11 uhci_hcd 0000:00:01.2: UHCI Host Controller uhci_hcd 0000:00:01.2: new USB bus registered, assigned bus number 1 uhci_hcd 0000:00:01.2: irq 11, io base 0x0000c020 usb usb1: New USB device found, idVendor=1d6b, idProduct=0001 usb usb1: New USB device strings: Mfr=3, Product=2, SerialNumber=1 usb usb1: Product: UHCI Host Controller usb usb1: Manufacturer: Linux 2.6.32-431.29.2.el6_lustre.g9835a2a.x86_64 uhci_hcd usb usb1: SerialNumber: 0000:00:01.2 usb usb1: configuration #1 chosen from 1 choice hub 1-0:1.0: USB hub found hub 1-0:1.0: 2 ports detected PNP: PS/2 Controller [PNP0303:KBD,PNP0f13:MOU] at 0x60,0x64 irq 1,12 serio: i8042 KBD port at 0x60,0x64 irq 1 serio: i8042 AUX port at 0x60,0x64 irq 12 mice: PS/2 mouse device common for all mice input: AT Translated Set 2 keyboard as /devices/platform/i8042/serio0/input/input2 rtc_cmos 00:01: RTC can wake from S4 rtc_cmos 00:01: rtc core: registered rtc_cmos as rtc0 rtc0: alarms up to one day, 114 bytes nvram cpuidle: using governor ladder cpuidle: using governor menu EFI Variables Facility v0.08 2004-May-17 usbcore: registered new interface driver hiddev usbcore: registered new interface driver usbhid usbhid: v2.6:USB HID core driver GRE over IPv4 demultiplexor driver TCP cubic registered Initializing XFRM netlink socket NET: Registered protocol family 17 registered taskstats version 1 rtc_cmos 00:01: setting system clock to 2014-10-27 04:57:52 UTC (1414385872) Initalizing network drop monitor service Freeing unused kernel memory: 1280k freed Write protecting the kernel read-only data: 10240k Freeing unused kernel memory: 792k freed Freeing unused kernel memory: 1584k freed dracut: dracut-004-336.el6_5.2 dracut: rd_NO_LUKS: removing cryptoluks activation dracut: rd_NO_LVM: removing LVM activation device-mapper: uevent: version 1.0.3 device-mapper: ioctl: 4.24.6-ioctl (2013-01-15) initialised: dm-devel@redhat.com udev: starting version 147 dracut: Starting plymouth daemon dracut: rd_NO_DM: removing DM RAID activation dracut: rd_NO_MD: removing MD RAID activation scsi0 : ata_piix scsi1 : ata_piix ata1: PATA max MWDMA2 cmd 0x1f0 ctl 0x3f6 bmdma 0xc000 irq 14 ata2: PATA max MWDMA2 cmd 0x170 ctl 0x376 bmdma 0xc008 irq 15 virtio-pci 0000:00:04.0: PCI INT A -> Link[LNKD] -> GSI 11 (level, high) -> IRQ 11 Refined TSC clocksource calibration: 2000.215 MHz. ACPI: PCI Interrupt Link [LNKA] enabled at IRQ 10 virtio-pci 0000:00:05.0: PCI INT A -> Link[LNKA] -> GSI 10 (level, high) -> IRQ 10 input: ImExPS/2 Generic Explorer Mouse as /devices/platform/i8042/serio1/input/input3 %G vda: vda1 vda2 vda3 vda4 EXT3-fs (vda1): recovery required on readonly filesystem EXT3-fs (vda1): write access will be enabled during recovery Clocksource tsc unstable (delta = 236268571 ns) kjournald starting. Commit interval 5 seconds EXT3-fs (vda1): recovery complete EXT3-fs (vda1): mounted filesystem with ordered data mode dracut: Mounted root filesystem /dev/vda1 SELinux: Disabled at runtime. type=1404 audit(1414385874.387:2): selinux=0 auid=4294967295 ses=4294967295 dracut: dracut: Switching root Welcome to CentOS Starting udev: udev: starting version 147 piix4_smbus 0000:00:01.3: SMBus Host Controller at 0xb100, revision 0 8139cp: 10/100 PCI Ethernet driver v1.3 (Mar 22, 2004) ACPI: PCI Interrupt Link [LNKC] enabled at IRQ 11 8139cp 0000:00:03.0: PCI INT A -> Link[LNKC] -> GSI 11 (level, high) -> IRQ 11 eth0: RTL-8139C+ at 0xffffc900009d4000, 00:16:3e:55:0a:70, IRQ 11 8139too Fast Ethernet driver 0.9.28 SPL: Loaded module v0.6.3-1 zunicode: module license 'CDDL' taints kernel. Disabling lock debugging due to kernel taint ZFS: Loaded module v0.6.3-1, ZFS pool version 5000, ZFS filesystem version 5 %Gplatform microcode: firmware: requesting amd-ucode/microcode_amd.bin microcode: CPU0: AMD CPU family 0x6 not supported Microcode Update Driver: v2.00 , Peter Oruba [ OK ] Setting hostname eagle-46vm1.eagle.hpdd.intel.com: [ OK ] Setting up Logical Volume Management: No volume groups found [ OK ] Checking filesystems Checking all file systems. [/sbin/fsck.ext3 (1) -- /] fsck.ext3 -a /dev/vda1 /dev/vda1: clean, 42985/1310720 files, 551834/5242880 blocks [ OK ] Remounting root filesystem in read-write mode: EXT3-fs (vda1): using internal journal [ OK ] Mounting local filesystems: [ OK ] Enabling local filesystem quotas: [ OK ] Enabling /etc/fstab swaps: Adding 4128764k swap on /dev/vda2. Priority:-1 extents:1 across:4128764k [ OK ] Entering non-interactive startup Calling the system activity data collector (sadc)... Mounting automounted ZFS filesystems: [ OK ] Exporting ZFS filesystems: [ OK ] Loading OpenIB kernel modules:NET: Registered protocol family 10 lo: Disabled Privacy Extensions [ OK ] Bringing up loopback interface: [ OK ] Bringing up interface eth0: Determining IP information for eth0...eth0: link up, 100Mbps, full-duplex, lpa 0x05E1 connect: Invalid argument done. [ OK ] Starting auditd: [ OK ] Starting system logger: [ OK ] Starting irqbalance: [ OK ] Starting rpcbind: [ OK ] Starting sssd: [ OK ] [ OK ] Starting NFS statd: [ OK ] No kdump initial ramdisk found.[WARNING] Rebuilding /boot/initrd-2.6.32-431.29.2.el6_lustre.g9835a2a.x86_64kdump.img RPC: Registered named UNIX socket transport module. RPC: Registered udp transport module. RPC: Registered tcp transport module. RPC: Registered tcp NFSv4.1 backchannel transport module. Slow work thread pool: Starting up Slow work thread pool: Ready FS-Cache: Loaded NFS: Registering the id_resolver key type FS-Cache: Netfs 'nfs' registered for caching mount.nfs: mounting 10.100.0.2:/export/scratch/dumps failed, reason given by server: No such file or directory /etc/kdump.conf: Bad NFS mount eagle-2.eagle.hpdd.intel.com:/export/scratch/dumps Failed to run mkdumprd Starting system message bus: [ OK ] Starting ibacm daemon:[ OK ] Mounting filesystems: [ OK ] Starting acpi daemon: [ OK ] Starting HAL daemon: [ OK ] Retrigger failed udev events[ OK ] Loading autofs4: [ OK ] Starting automount: [ OK ] Installing knfsd (copyright (C) 1996 okir@monad.swb.de). Starting NFS services: [ OK ] Starting NFS quotas: [ OK ] Starting NFS mountd: [ OK ] Starting NFS daemon: NFSD: Using /var/lib/nfs/v4recovery as the NFSv4 state recovery directory NFSD: starting 90-second grace period [ OK ] Starting RPC idmapd: [ OK ] Starting MUNGE: munged[ OK ] Starting sshd: [ OK ] Starting xinetd: [ OK ] Starting ntpd: [ OK ] Starting postfix: [ OK ] Starting crond: [ OK ] Starting atd: [ OK ] Starting certmonger: [ OK ] CentOS release 6.5 (Final) Kernel 2.6.32-431.29.2.el6_lustre.g9835a2a.x86_64 on an x86_64 eagle-46vm1.eagle.hpdd.intel.com login: SPL: using hostid 0x00000000 LNet: HW CPU cores: 1, npartitions: 1 alg: No test for crc32 (crc32-table) alg: No test for adler32 (adler32-zlib) padlock: VIA PadLock Hash Engine not detected. Lustre: Lustre: Build Version: jenkins-arch=x86_64,build_type=server,distro=el6,ib_stack=inkernel-96-g3602066-PRISTINE-2.6.32-431.29.2.el6_lustre.g9835a2a.x86_64 LNet: Added LNI 10.100.4.140@tcp [8/256/0/180] LNet: Accept secure, port 988 Lustre: lustre-OST0000: new disk, initializing Lustre: srv-lustre-OST0000: No data found on store. Initialize space Lustre: DEBUG MARKER: PATH=/usr/lib64/lustre/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:/opt/iozone/bin:/usr/lib64/lustre/tests/mpi:/usr/lib64/lustre/tests/racer:/usr/lib64/lustre/../lustre-iokit/sgpdd-survey:/usr/lib64/lustre/tests:/usr/lib64/lustre/utils/gss:/usr/li Lustre: DEBUG MARKER: zfs get -H -o value lustre:svname lustre-ost1/ost1 2>/dev/null Lustre: DEBUG MARKER: mkdir -p /mnt/ost2 Lustre: DEBUG MARKER: zpool list -H lustre-ost2 >/dev/null 2>&1 || zpool import -f -o cachefile=none -d /dev lustre-ost2 Lustre: DEBUG MARKER: mkdir -p /mnt/ost2; mount -t lustre lustre-ost2/ost2 /mnt/ost2 Lustre: lustre-OST0001: new disk, initializing Lustre: srv-lustre-OST0001: No data found on store. Initialize space Lustre: DEBUG MARKER: PATH=/usr/lib64/lustre/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:/opt/iozone/bin:/usr/lib64/lustre/tests/mpi:/usr/lib64/lustre/tests/racer:/usr/lib64/lustre/../lustre-iokit/sgpdd-survey:/usr/lib64/lustre/tests:/usr/lib64/lustre/utils/gss:/usr/li Lustre: DEBUG MARKER: zfs get -H -o value lustre:svname lustre-ost2/ost2 2>/dev/null Lustre: DEBUG MARKER: /usr/sbin/lctl mark Using TIMEOUT=20 Lustre: DEBUG MARKER: Using TIMEOUT=20 Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-zfs.lustre-OST0000.quota_slave.enabled Lustre: DEBUG MARKER: PATH=/usr/lib64/lustre/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:/opt/iozone/bin:/usr/lib64/lustre/tests/mpi:/usr/lib64/lustre/tests/racer:/usr/lib64/lustre/../lustre-iokit/sgpdd-survey:/usr/lib64/lustre/tests:/usr/lib64/lustre/utils/gss:/usr/li Lustre: DEBUG MARKER: uname -n Lustre: DEBUG MARKER: PATH=/usr/lib64/lustre/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:/opt/iozone/bin:/usr/lib64/lustre/tests/mpi:/usr/lib64/lustre/tests/racer:/usr/lib64/lustre/../lustre-iokit/sgpdd-survey:/usr/lib64/lustre/tests:/usr/lib64/lustre/utils/gss:/usr/li Lustre: DEBUG MARKER: /usr/sbin/lctl mark -----============= acceptance-small: sanity ============----- Sun Oct 26 22:11:59 PDT 2014 Lustre: DEBUG MARKER: -----============= acceptance-small: sanity ============----- Sun Oct 26 22:11:59 PDT 2014 Lustre: DEBUG MARKER: running=$(grep -c /mnt/ost1' ' /proc/mounts); mpts=$(mount | grep -c /mnt/ost1' '); if [ $running -ne $mpts ]; then echo $(hostname) env are INSANE!; exit 1; fi Lustre: DEBUG MARKER: running=$(grep -c /mnt/ost2' ' /proc/mounts); mpts=$(mount | grep -c /mnt/ost2' '); if [ $running -ne $mpts ]; then echo $(hostname) env are INSANE!; exit 1; fi Lustre: DEBUG MARKER: zfs get -H -o value lustre:svname lustre-ost1/ost1 2>/dev/null Lustre: DEBUG MARKER: cat /proc/mounts Lustre: DEBUG MARKER: zfs get -H -o value lustre:svname lustre-ost2/ost2 2>/dev/null Lustre: DEBUG MARKER: cat /proc/mounts Lustre: DEBUG MARKER: /usr/sbin/lctl mark Using TIMEOUT=20 Lustre: DEBUG MARKER: Using TIMEOUT=20 Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-zfs.lustre-OST0000.quota_slave.enabled Lustre: DEBUG MARKER: PATH=/usr/lib64/lustre/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:/opt/iozone/bin:/opt/iozone/bin:/usr/lib64/lustre/tests/mpi:/usr/lib64/lustre/tests/racer:/usr/lib64/lustre/../lustre-iokit/sgpdd-survey:/usr/lib64/lustre/tests:/usr/lib64/lustre/u Lustre: DEBUG MARKER: /usr/sbin/lctl mark excepting tests: 76 42a 42b 42c 42d 45 51d 68b 34h 40 48a 180 184c Lustre: DEBUG MARKER: excepting tests: 76 42a 42b 42c 42d 45 51d 68b 34h 40 48a 180 184c Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 0a: touch\; rm ======================= 22:12:15 $1414386735$ Lustre: DEBUG MARKER: == sanity test 0a: touch; rm ======================= 22:12:15 (1414386735) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 0b: chmod 0755 \/mnt\/lustre =============================== 22:12:16 $1414386736$ Lustre: DEBUG MARKER: == sanity test 0b: chmod 0755 /mnt/lustre =============================== 22:12:16 (1414386736) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 0c: check import proc =============================== 22:12:17 $1414386737$ Lustre: DEBUG MARKER: == sanity test 0c: check import proc =============================== 22:12:17 (1414386737) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 1: mkdir\; remkdir\; rmdir ================================ 22:12:18 $1414386738$ Lustre: DEBUG MARKER: == sanity test 1: mkdir; remkdir; rmdir ================================ 22:12:18 (1414386738) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 2: mkdir\; touch\; rmdir\; check file ====================== 22:12:20 $1414386740$ Lustre: DEBUG MARKER: == sanity test 2: mkdir; touch; rmdir; check file ====================== 22:12:20 (1414386740) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 3: mkdir\; touch\; rmdir\; check dir ======================= 22:12:21 $1414386741$ Lustre: DEBUG MARKER: == sanity test 3: mkdir; touch; rmdir; check dir ======================= 22:12:21 (1414386741) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 4: mkdir\; touch dir\/file\; rmdir\; checkdir $expect error$ == 22:12:22 $1414386742$ Lustre: DEBUG MARKER: == sanity test 4: mkdir; touch dir/file; rmdir; checkdir (expect error) == 22:12:22 (1414386742) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 5: mkdir ...\/d5 ...\/d5\/d2\; chmod ...\/d5\/d2 ============== 22:12:23 $1414386743$ Lustre: DEBUG MARKER: == sanity test 5: mkdir .../d5 .../d5/d2; chmod .../d5/d2 ============== 22:12:23 (1414386743) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 6a: touch f6a\; chmod f6a\; runas -u 500 -g 500 chmod f6a $should return error$ ==== 22:12:25 $1414386745$ Lustre: DEBUG MARKER: == sanity test 6a: touch f6a; chmod f6a; runas -u 500 -g 500 chmod f6a (should return error) ==== 22:12:25 (1414386745) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 6c: touch f6c\; chown f6c\; runas -u 500 -g 500 chown f6c $should return error$ ==== 22:12:26 $1414386746$ Lustre: DEBUG MARKER: == sanity test 6c: touch f6c; chown f6c; runas -u 500 -g 500 chown f6c (should return error) ==== 22:12:26 (1414386746) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 6e: touch f6e\; chgrp f6e\; runas -u 500 -g 500 chgrp f6e $should return error$ ==== 22:12:28 $1414386748$ Lustre: DEBUG MARKER: == sanity test 6e: touch f6e; chgrp f6e; runas -u 500 -g 500 chgrp f6e (should return error) ==== 22:12:28 (1414386748) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 6g: Is new dir in sgid dir inheriting group? == 22:12:29 $1414386749$ Lustre: DEBUG MARKER: == sanity test 6g: Is new dir in sgid dir inheriting group? == 22:12:29 (1414386749) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 6h: runas -u 500 -g 500 chown RUNAS_ID.0 ...\/f6h $should return error$ == 22:12:31 $1414386751$ Lustre: DEBUG MARKER: == sanity test 6h: runas -u 500 -g 500 chown RUNAS_ID.0 .../f6h (should return error) == 22:12:31 (1414386751) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 7a: mkdir ...\/d7\; mcreate ...\/d7\/f\; chmod ...\/d7\/f ====== 22:12:32 $1414386752$ Lustre: DEBUG MARKER: == sanity test 7a: mkdir .../d7; mcreate .../d7/f; chmod .../d7/f ====== 22:12:32 (1414386752) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 7b: mkdir ...\/d7\; mcreate d7\/f2\; echo foo \> d7\/f2 ======= 22:12:33 $1414386753$ Lustre: DEBUG MARKER: == sanity test 7b: mkdir .../d7; mcreate d7/f2; echo foo > d7/f2 ======= 22:12:33 (1414386753) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 8: mkdir ...\/d8\; touch ...\/d8\/f\; chmod ...\/d8\/f ========= 22:12:34 $1414386754$ Lustre: DEBUG MARKER: == sanity test 8: mkdir .../d8; touch .../d8/f; chmod .../d8/f ========= 22:12:34 (1414386754) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 9: mkdir ...\/d9 ...\/d9\/d2 ...\/d9\/d2\/d3 ================== 22:12:35 $1414386755$ Lustre: DEBUG MARKER: == sanity test 9: mkdir .../d9 .../d9/d2 .../d9/d2/d3 ================== 22:12:35 (1414386755) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 10: mkdir ...\/d10 ...\/d10\/d2\; touch ...\/d10\/d2\/f ======== 22:12:37 $1414386757$ Lustre: DEBUG MARKER: == sanity test 10: mkdir .../d10 .../d10/d2; touch .../d10/d2/f ======== 22:12:37 (1414386757) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 11: mkdir ...\/d11 d11\/d2\; chmod ...\/d11\/d2 ============== 22:12:38 $1414386758$ Lustre: DEBUG MARKER: == sanity test 11: mkdir .../d11 d11/d2; chmod .../d11/d2 ============== 22:12:38 (1414386758) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 12: touch ...\/d12\/f\; chmod ...\/d12\/f ...\/d12\/f ========== 22:12:40 $1414386760$ Lustre: DEBUG MARKER: == sanity test 12: touch .../d12/f; chmod .../d12/f .../d12/f ========== 22:12:40 (1414386760) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 13: creat ...\/d13\/f\; dd ...\/d13\/f\; \> ...\/d13\/f ========== 22:12:41 $1414386761$ Lustre: DEBUG MARKER: == sanity test 13: creat .../d13/f; dd .../d13/f; > .../d13/f ========== 22:12:41 (1414386761) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 14: touch ...\/d14\/f\; rm ...\/d14\/f\; rm ...\/d14\/f ========= 22:12:42 $1414386762$ Lustre: DEBUG MARKER: == sanity test 14: touch .../d14/f; rm .../d14/f; rm .../d14/f ========= 22:12:42 (1414386762) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 15: touch ...\/d15\/f\; mv ...\/d15\/f ...\/d15\/f2 ============ 22:12:44 $1414386764$ Lustre: DEBUG MARKER: == sanity test 15: touch .../d15/f; mv .../d15/f .../d15/f2 ============ 22:12:44 (1414386764) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 16: touch ...\/d16\/f\; rm -rf ...\/d16\/f =================== 22:12:45 $1414386765$ Lustre: DEBUG MARKER: == sanity test 16: touch .../d16/f; rm -rf .../d16/f =================== 22:12:45 (1414386765) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 17a: symlinks: create, remove $real$ ==================== 22:12:46 $1414386766$ Lustre: DEBUG MARKER: == sanity test 17a: symlinks: create, remove (real) ==================== 22:12:46 (1414386766) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 17b: symlinks: create, remove $dangling$ ================ 22:12:47 $1414386767$ Lustre: DEBUG MARKER: == sanity test 17b: symlinks: create, remove (dangling) ================ 22:12:47 (1414386767) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 17c: symlinks: open dangling $should return error$ ====== 22:12:49 $1414386769$ Lustre: DEBUG MARKER: == sanity test 17c: symlinks: open dangling (should return error) ====== 22:12:49 (1414386769) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 17d: symlinks: create dangling ========================== 22:12:50 $1414386770$ Lustre: DEBUG MARKER: == sanity test 17d: symlinks: create dangling ========================== 22:12:50 (1414386770) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 17e: symlinks: create recursive symlink $should return error$ ====== 22:12:51 $1414386771$ Lustre: DEBUG MARKER: == sanity test 17e: symlinks: create recursive symlink (should return error) ====== 22:12:51 (1414386771) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 17f: symlinks: long and very long symlink name ========================== 22:12:52 $1414386772$ Lustre: DEBUG MARKER: == sanity test 17f: symlinks: long and very long symlink name ========================== 22:12:52 (1414386772) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 17g: symlinks: really long symlink name and inode boundaries == 22:12:54 $1414386774$ Lustre: DEBUG MARKER: == sanity test 17g: symlinks: really long symlink name and inode boundaries == 22:12:54 (1414386774) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 17h: create objects: lov_free_memmd doesn\'t lbug == 22:12:57 $1414386777$ Lustre: DEBUG MARKER: == sanity test 17h: create objects: lov_free_memmd() doesn't lbug == 22:12:57 (1414386777) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 17i: don\'t panic on short symlink == 22:12:58 $1414386778$ Lustre: DEBUG MARKER: == sanity test 17i: don't panic on short symlink == 22:12:58 (1414386778) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 17k: symlinks: rsync with xattrs enabled =========================== 22:13:00 $1414386780$ Lustre: DEBUG MARKER: == sanity test 17k: symlinks: rsync with xattrs enabled =========================== 22:13:00 (1414386780) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 17l: Ensure lgetxattr\'s returned xattr size is consistent ========== 22:13:01 $1414386781$ Lustre: DEBUG MARKER: == sanity test 17l: Ensure lgetxattr's returned xattr size is consistent ========== 22:13:01 (1414386781) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 17m: run e2fsck against MDT which contains short\/long symlink == 22:13:03 $1414386783$ Lustre: DEBUG MARKER: == sanity test 17m: run e2fsck against MDT which contains short/long symlink == 22:13:03 (1414386783) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_17m only for ldiskfs MDT Lustre: DEBUG MARKER: SKIP: sanity test_17m only for ldiskfs MDT Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 17n: run e2fsck against master\/slave MDT which contains remote dir == 22:13:05 $1414386785$ Lustre: DEBUG MARKER: == sanity test 17n: run e2fsck against master/slave MDT which contains remote dir == 22:13:05 (1414386785) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_17n only for ldiskfs MDT Lustre: DEBUG MARKER: SKIP: sanity test_17n only for ldiskfs MDT Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 17o: stat file with incompat LMA feature == 22:13:07 $1414386787$ Lustre: DEBUG MARKER: == sanity test 17o: stat file with incompat LMA feature == 22:13:07 (1414386787) LustreError: 11-0: lustre-MDT0000-lwp-OST0000: Communicating with 10.100.4.147@tcp, operation obd_ping failed with -107. Lustre: lustre-MDT0000-lwp-OST0000: Connection to lustre-MDT0000 (at 10.100.4.147@tcp) was lost; in progress operations using this service will wait for recovery to complete LustreError: 11-0: MGC10.100.4.147@tcp: Communicating with 10.100.4.147@tcp, operation obd_ping failed with -107. LustreError: Skipped 1 previous similar message LustreError: 166-1: MGC10.100.4.147@tcp: Connection to MGS (at 10.100.4.147@tcp) was lost; in progress operations using this service will fail Lustre: Evicted from MGS (at 10.100.4.147@tcp) after server handle changed from 0xe51f1a4c383ec4dc to 0xe51f1a4c383eea83 Lustre: MGC10.100.4.147@tcp: Connection restored to MGS (at 10.100.4.147@tcp) Lustre: lustre-OST0000: deleting orphan objects from 0x0:14 to 0x0:33 Lustre: lustre-OST0001: deleting orphan objects from 0x0:15 to 0x0:33 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 18: touch ...\/f \; ls ... ================================ 22:13:32 $1414386812$ Lustre: DEBUG MARKER: == sanity test 18: touch .../f ; ls ... ================================ 22:13:32 (1414386812) Lustre: 8984:0:(client.c:1918:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1414386798/real 1414386798] req@ffff8800717ebc00 x1483091918979824/t0(0) o38->lustre-MDT0000-lwp-OST0000@10.100.4.147@tcp:12/10 lens 400/544 e 0 to 1 dl 1414386809 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 19a: touch ...\/f19 \; ls -l ... \; rm ...\/f19 ============= 22:13:33 $1414386813$ Lustre: DEBUG MARKER: == sanity test 19a: touch .../f19 ; ls -l ... ; rm .../f19 ============= 22:13:33 (1414386813) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 19b: ls -l ...\/f19 $should return error$ ================ 22:13:35 $1414386815$ Lustre: DEBUG MARKER: == sanity test 19b: ls -l .../f19 (should return error) ================ 22:13:35 (1414386815) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 19c: runas -u 500 -g 500 touch ...\/f19 $should return error$ ==== 22:13:36 $1414386816$ Lustre: DEBUG MARKER: == sanity test 19c: runas -u 500 -g 500 touch .../f19 (should return error) ==== 22:13:36 (1414386816) LustreError: 167-0: lustre-MDT0000-lwp-OST0000: This client was evicted by lustre-MDT0000; in progress operations using this service will fail. Lustre: lustre-MDT0000-lwp-OST0000: Connection restored to lustre-MDT0000 (at 10.100.4.147@tcp) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 19d: cat ...\/f19 $should return error$ ================ 22:13:37 $1414386817$ Lustre: DEBUG MARKER: == sanity test 19d: cat .../f19 (should return error) ================ 22:13:37 (1414386817) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 20: touch ...\/f \; ls -l ... ============================= 22:13:38 $1414386818$ Lustre: DEBUG MARKER: == sanity test 20: touch .../f ; ls -l ... ============================= 22:13:38 (1414386818) Lustre: DEBUG MARKER: /usr/sbin/lctl mark 1 done Lustre: DEBUG MARKER: 1 done Lustre: DEBUG MARKER: /usr/sbin/lctl mark 2 done Lustre: DEBUG MARKER: 2 done Lustre: DEBUG MARKER: /usr/sbin/lctl mark 3 done Lustre: DEBUG MARKER: 3 done Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 21: write to dangling link ============================== 22:13:41 $1414386821$ Lustre: DEBUG MARKER: == sanity test 21: write to dangling link ============================== 22:13:41 (1414386821) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 22: unpack tar archive as non-root user ================= 22:13:42 $1414386822$ Lustre: DEBUG MARKER: == sanity test 22: unpack tar archive as non-root user ================= 22:13:42 (1414386822) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 23a: O_CREAT\|O_EXCL in subdir ============================ 22:13:44 $1414386824$ Lustre: DEBUG MARKER: == sanity test 23a: O_CREAT|O_EXCL in subdir ============================ 22:13:44 (1414386824) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 23b: O_APPEND check ============================ 22:13:45 $1414386825$ Lustre: DEBUG MARKER: == sanity test 23b: O_APPEND check ============================ 22:13:45 (1414386825) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24a: rename file to non-existent target == 22:13:46 $1414386826$ Lustre: DEBUG MARKER: == sanity test 24a: rename file to non-existent target == 22:13:46 (1414386826) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24b: rename file to existing target == 22:13:48 $1414386828$ Lustre: DEBUG MARKER: == sanity test 24b: rename file to existing target == 22:13:48 (1414386828) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24c: rename directory to non-existent target == 22:13:49 $1414386829$ Lustre: DEBUG MARKER: == sanity test 24c: rename directory to non-existent target == 22:13:49 (1414386829) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24d: rename directory to existing target == 22:13:50 $1414386830$ Lustre: DEBUG MARKER: == sanity test 24d: rename directory to existing target == 22:13:50 (1414386830) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24e: touch ...\/R5a\/f\; rename ...\/R5a\/f ...\/R5b\/g ======== 22:13:51 $1414386831$ Lustre: DEBUG MARKER: == sanity test 24e: touch .../R5a/f; rename .../R5a/f .../R5b/g ======== 22:13:51 (1414386831) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24f: touch ...\/R6a\/f R6b\/g\; mv ...\/R6a\/f ...\/R6b\/g ====== 22:13:53 $1414386833$ Lustre: DEBUG MARKER: == sanity test 24f: touch .../R6a/f R6b/g; mv .../R6a/f .../R6b/g ====== 22:13:53 (1414386833) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24g: mkdir ...\/R7{a,b}\/d\; mv ...\/R7a\/d ...\/R7b\/e ======== 22:13:54 $1414386834$ Lustre: DEBUG MARKER: == sanity test 24g: mkdir .../R7a/d; .../R7b/d; mv .../R7a/d .../R7b/e ======== 22:13:54 (1414386834) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24h: mkdir ...\/R8{a,b}\/{d,e}\; rename ...\/R8a\/d ...\/R8b\/e == 22:13:55 $1414386835$ Lustre: DEBUG MARKER: == sanity test 24h: mkdir .../R8a/d; .../R8a/e; .../R8b/d; .../R8b/e; rename .../R8a/d .../R8b/e == 22:13:55 (1414386835) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24i: rename file to dir error: touch f \; mkdir a \; rename f a == 22:13:56 $1414386836$ Lustre: DEBUG MARKER: == sanity test 24i: rename file to dir error: touch f ; mkdir a ; rename f a == 22:13:56 (1414386836) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24j: source does not exist ============================== 22:13:58 $1414386838$ Lustre: DEBUG MARKER: == sanity test 24j: source does not exist ============================== 22:13:58 (1414386838) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24k: touch ...\/R11a\/f\; mv ...\/R11a\/f ...\/R11a\/d ========= 22:13:59 $1414386839$ Lustre: DEBUG MARKER: == sanity test 24k: touch .../R11a/f; mv .../R11a/f .../R11a/d ========= 22:13:59 (1414386839) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24l: Renaming a file to itself ========================== 22:14:00 $1414386840$ Lustre: DEBUG MARKER: == sanity test 24l: Renaming a file to itself ========================== 22:14:00 (1414386840) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24m: Renaming a file to a hard link to itself =========== 22:14:01 $1414386841$ Lustre: DEBUG MARKER: == sanity test 24m: Renaming a file to a hard link to itself =========== 22:14:01 (1414386841) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24n: Statting the old file after renaming $Posix rename 2$ == 22:14:02 $1414386842$ Lustre: DEBUG MARKER: == sanity test 24n: Statting the old file after renaming (Posix rename 2) == 22:14:02 (1414386842) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24o: rename of files during htree split ================= 22:14:04 $1414386844$ Lustre: DEBUG MARKER: == sanity test 24o: rename of files during htree split ================= 22:14:04 (1414386844) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24p: mkdir ...\/R12{a,b}\; rename ...\/R12a ...\/R12b == 22:17:17 $1414387037$ Lustre: DEBUG MARKER: == sanity test 24p: mkdir .../R12a; .../R12b; rename .../R12a .../R12b == 22:17:17 (1414387037) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24q: mkdir ...\/R13{a,b}\; open R13b rename R13a R13b ===== 22:17:18 $1414387038$ Lustre: DEBUG MARKER: == sanity test 24q: mkdir .../R13a; .../R13b; open R13b rename R13a R13b ===== 22:17:18 (1414387038) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24r: mkdir ...\/R14a\/b\; rename ...\/R14a ...\/R14a\/b ======= 22:17:20 $1414387040$ Lustre: DEBUG MARKER: == sanity test 24r: mkdir .../R14a/b; rename .../R14a .../R14a/b ======= 22:17:20 (1414387040) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24s: mkdir ...\/R15a\/b\/c\; rename ...\/R15a ...\/R15a\/b\/c === 22:17:21 $1414387041$ Lustre: DEBUG MARKER: == sanity test 24s: mkdir .../R15a/b/c; rename .../R15a .../R15a/b/c === 22:17:21 (1414387041) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24t: mkdir ...\/R16a\/b\/c\; rename ...\/R16a\/b\/c ...\/R16a === 22:17:22 $1414387042$ Lustre: DEBUG MARKER: == sanity test 24t: mkdir .../R16a/b/c; rename .../R16a/b/c .../R16a === 22:17:22 (1414387042) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24u: create stripe file == 22:17:23 $1414387043$ Lustre: DEBUG MARKER: == sanity test 24u: create stripe file == 22:17:23 (1414387043) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24v: list directory with large files $handle hash collision, bug: 17560$ == 22:17:25 $1414387045$ Lustre: DEBUG MARKER: == sanity test 24v: list directory with large files (handle hash collision, bug: 17560) == 22:17:25 (1414387045) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24w: Reading a file larger than 4Gb == 22:18:53 $1414387133$ Lustre: DEBUG MARKER: == sanity test 24w: Reading a file larger than 4Gb == 22:18:53 (1414387133) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24x: cross rename\/link should be failed == 22:18:54 $1414387134$ Lustre: DEBUG MARKER: == sanity test 24x: cross rename/link should be failed == 22:18:54 (1414387134) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_24x needs \>= 2 MDTs Lustre: DEBUG MARKER: SKIP: sanity test_24x needs >= 2 MDTs Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24y: rename\/link on the same dir should succeed == 22:18:56 $1414387136$ Lustre: DEBUG MARKER: == sanity test 24y: rename/link on the same dir should succeed == 22:18:56 (1414387136) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_24y needs \>= 2 MDTs Lustre: DEBUG MARKER: SKIP: sanity test_24y needs >= 2 MDTs Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24z: rename one remote dir to another remote dir should fail == 22:18:58 $1414387138$ Lustre: DEBUG MARKER: == sanity test 24z: rename one remote dir to another remote dir should fail == 22:18:58 (1414387138) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_24z needs \>= 2 MDTs Lustre: DEBUG MARKER: SKIP: sanity test_24z needs >= 2 MDTs Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 24A: readdir returns correct number of entries. == 22:18:59 $1414387139$ Lustre: DEBUG MARKER: == sanity test 24A: readdir() returns correct number of entries. == 22:18:59 (1414387139) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 25a: create file in symlinked directory ================= 22:19:42 $1414387182$ Lustre: DEBUG MARKER: == sanity test 25a: create file in symlinked directory ================= 22:19:42 (1414387182) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 25b: lookup file in symlinked directory ================= 22:19:43 $1414387183$ Lustre: DEBUG MARKER: == sanity test 25b: lookup file in symlinked directory ================= 22:19:43 (1414387183) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 26a: multiple component symlink ========================= 22:19:45 $1414387185$ Lustre: DEBUG MARKER: == sanity test 26a: multiple component symlink ========================= 22:19:45 (1414387185) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 26b: multiple component symlink at end of lookup ======== 22:19:46 $1414387186$ Lustre: DEBUG MARKER: == sanity test 26b: multiple component symlink at end of lookup ======== 22:19:46 (1414387186) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 26c: chain of symlinks ================================== 22:19:47 $1414387187$ Lustre: DEBUG MARKER: == sanity test 26c: chain of symlinks ================================== 22:19:47 (1414387187) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 26d: create multiple component recursive symlink ======== 22:19:48 $1414387188$ Lustre: DEBUG MARKER: == sanity test 26d: create multiple component recursive symlink ======== 22:19:48 (1414387188) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 26e: unlink multiple component recursive symlink ======== 22:19:50 $1414387190$ Lustre: DEBUG MARKER: == sanity test 26e: unlink multiple component recursive symlink ======== 22:19:50 (1414387190) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 26f: rm -r of a directory which has recursive symlink === 22:19:51 $1414387191$ Lustre: DEBUG MARKER: == sanity test 26f: rm -r of a directory which has recursive symlink === 22:19:51 (1414387191) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27a: one stripe file ==================================== 22:19:52 $1414387192$ Lustre: DEBUG MARKER: == sanity test 27a: one stripe file ==================================== 22:19:52 (1414387192) Lustre: DEBUG MARKER: /usr/sbin/lctl mark == test_27a: write to one stripe file ========================= Lustre: DEBUG MARKER: == test_27a: write to one stripe file ========================= Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27b: create and write to two stripe file == 22:19:54 $1414387194$ Lustre: DEBUG MARKER: == sanity test 27b: create and write to two stripe file == 22:19:54 (1414387194) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27d: create file with default settings ================== 22:19:55 $1414387195$ Lustre: DEBUG MARKER: == sanity test 27d: create file with default settings ================== 22:19:55 (1414387195) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27e: setstripe existing file $should return error$ ======== 22:19:56 $1414387196$ Lustre: DEBUG MARKER: == sanity test 27e: setstripe existing file (should return error) ======== 22:19:56 (1414387196) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27f: setstripe with bad stripe size $should return error$ == 22:19:58 $1414387198$ Lustre: DEBUG MARKER: == sanity test 27f: setstripe with bad stripe size (should return error) == 22:19:58 (1414387198) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27g: \/usr\/bin\/lfs getstripe with no objects == 22:19:59 $1414387199$ Lustre: DEBUG MARKER: == sanity test 27g: /usr/bin/lfs getstripe with no objects == 22:19:59 (1414387199) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27i: \/usr\/bin\/lfs getstripe with some objects == 22:20:00 $1414387200$ Lustre: DEBUG MARKER: == sanity test 27i: /usr/bin/lfs getstripe with some objects == 22:20:00 (1414387200) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27j: setstripe with bad stripe offset $should return error$ == 22:20:01 $1414387201$ Lustre: DEBUG MARKER: == sanity test 27j: setstripe with bad stripe offset (should return error) == 22:20:01 (1414387201) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27k: limit i_blksize for broken user apps =============== 22:20:03 $1414387203$ Lustre: DEBUG MARKER: == sanity test 27k: limit i_blksize for broken user apps =============== 22:20:03 (1414387203) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27l: check setstripe permissions $should return error$ == 22:20:04 $1414387204$ Lustre: DEBUG MARKER: == sanity test 27l: check setstripe permissions (should return error) == 22:20:04 (1414387204) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27m: create file while OST0 was full ==================== 22:20:05 $1414387205$ Lustre: DEBUG MARKER: == sanity test 27m: create file while OST0 was full ==================== 22:20:05 (1414387205) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_27m 130568064 \> 400000 skipping out-of-space test on OST0 Lustre: DEBUG MARKER: SKIP: sanity test_27m 130568064 > 400000 skipping out-of-space test on OST0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27n: create file with some full OSTs ==================== 22:20:07 $1414387207$ Lustre: DEBUG MARKER: == sanity test 27n: create file with some full OSTs ==================== 22:20:07 (1414387207) Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param fail_val=0 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x215 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x80000215 Lustre: *** cfs_fail_loc=215, val=0*** Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27o: create file with all full OSTs $should error$ ====== 22:20:45 $1414387245$ Lustre: DEBUG MARKER: == sanity test 27o: create file with all full OSTs (should error) ====== 22:20:45 (1414387245) Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param fail_val=-1 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x215 Lustre: *** cfs_fail_loc=215, val=1*** Lustre: DEBUG MARKER: lctl set_param fail_loc=0x215 Lustre: *** cfs_fail_loc=215, val=1*** Lustre: Skipped 1 previous similar message Lustre: *** cfs_fail_loc=215, val=1*** Lustre: Skipped 1 previous similar message Lustre: DEBUG MARKER: lctl set_param fail_val=-1 Lustre: *** cfs_fail_loc=215, val=1*** Lustre: Skipped 5 previous similar messages Lustre: DEBUG MARKER: lctl set_param fail_loc=0x215 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x215 Lustre: *** cfs_fail_loc=215, val=1*** Lustre: Skipped 3 previous similar messages Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27p: append to a truncated file with some full OSTs ===== 22:21:40 $1414387300$ Lustre: DEBUG MARKER: == sanity test 27p: append to a truncated file with some full OSTs ===== 22:21:40 (1414387300) Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param fail_val=0 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x215 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x80000215 Lustre: *** cfs_fail_loc=215, val=0*** Lustre: Skipped 1 previous similar message Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27q: append to truncated file with all OSTs full $should error$ ===== 22:22:18 $1414387338$ Lustre: DEBUG MARKER: == sanity test 27q: append to truncated file with all OSTs full (should error) ===== 22:22:18 (1414387338) Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param fail_val=-1 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x215 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x215 Lustre: *** cfs_fail_loc=215, val=1*** Lustre: DEBUG MARKER: lctl set_param fail_val=-1 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x215 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x215 Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27r: stripe file with some full OSTs $shouldn\'t LBUG$ === 22:23:12 $1414387392$ Lustre: DEBUG MARKER: == sanity test 27r: stripe file with some full OSTs (shouldn't LBUG) === 22:23:12 (1414387392) Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param fail_val=0 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x215 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x80000215 Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27s: lsm_xfersize overflow $should error$ $bug 10725$ == 22:23:50 $1414387430$ Lustre: DEBUG MARKER: == sanity test 27s: lsm_xfersize overflow (should error) (bug 10725) == 22:23:50 (1414387430) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27t: check that utils parse path correctly == 22:23:51 $1414387431$ Lustre: DEBUG MARKER: == sanity test 27t: check that utils parse path correctly == 22:23:51 (1414387431) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27u: skip object creation on OSC w\/o objects ============ 22:23:53 $1414387433$ Lustre: DEBUG MARKER: == sanity test 27u: skip object creation on OSC w/o objects ============ 22:23:53 (1414387433) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27v: skip object creation on slow OST =================== 22:24:17 $1414387457$ Lustre: DEBUG MARKER: == sanity test 27v: skip object creation on slow OST =================== 22:24:17 (1414387457) Lustre: DEBUG MARKER: lctl set_param fail_val=-1 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x215 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x215 Lustre: *** cfs_fail_loc=215, val=1*** Lustre: Skipped 12 previous similar messages Lustre: *** cfs_fail_loc=215, val=0*** Lustre: DEBUG MARKER: lctl set_param fail_val=-1 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x215 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x215 Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x705 Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27w: check \/usr\/bin\/lfs setstripe -S option == 22:25:21 $1414387521$ Lustre: DEBUG MARKER: == sanity test 27w: check /usr/bin/lfs setstripe -S option == 22:25:21 (1414387521) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27wa: check \/usr\/bin\/lfs setstripe -c -i options == 22:25:22 $1414387522$ Lustre: DEBUG MARKER: == sanity test 27wa: check /usr/bin/lfs setstripe -c -i options == 22:25:22 (1414387522) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27x: create files while OST0 is degraded == 22:25:24 $1414387524$ Lustre: DEBUG MARKER: == sanity test 27x: create files while OST0 is degraded == 22:25:24 (1414387524) Lustre: DEBUG MARKER: lctl set_param -n obdfilter.lustre-OST0000.degraded 1 Lustre: DEBUG MARKER: lctl set_param -n obdfilter.lustre-OST0000.degraded 0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27y: create files while OST0 is degraded and the rest inactive == 22:25:36 $1414387536$ Lustre: DEBUG MARKER: == sanity test 27y: create files while OST0 is degraded and the rest inactive == 22:25:36 (1414387536) Lustre: DEBUG MARKER: lctl set_param -n obdfilter.lustre-OST0000.degraded=1 Lustre: DEBUG MARKER: lctl set_param -n obdfilter.lustre-OST0000.degraded=0 Lustre: lustre-OST0001: Client lustre-MDT0000-mdtlov_UUID (at 10.100.4.147@tcp) reconnecting Lustre: lustre-OST0001: deleting orphan objects from 0x0:1174 to 0x0:1217 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27z: check SEQ\/OID on the MDT and OST filesystems == 22:26:00 $1414387560$ Lustre: DEBUG MARKER: == sanity test 27z: check SEQ/OID on the MDT and OST filesystems == 22:26:00 (1414387560) Lustre: DEBUG MARKER: /usr/sbin/lctl mark check file \/mnt\/lustre\/d27z.sanity\/f27z.sanity-1 Lustre: DEBUG MARKER: check file /mnt/lustre/d27z.sanity/f27z.sanity-1 Lustre: DEBUG MARKER: /usr/sbin/lctl mark FID seq 0x200000400, oid 0x67fd ver 0x0 Lustre: DEBUG MARKER: FID seq 0x200000400, oid 0x67fd ver 0x0 Lustre: DEBUG MARKER: /usr/sbin/lctl mark LOV seq 0x200000400, oid 0x67fd, count: 1 Lustre: DEBUG MARKER: LOV seq 0x200000400, oid 0x67fd, count: 1 Lustre: DEBUG MARKER: /usr/sbin/lctl mark check file \/mnt\/lustre\/d27z.sanity\/f27z.sanity-2 Lustre: DEBUG MARKER: check file /mnt/lustre/d27z.sanity/f27z.sanity-2 Lustre: DEBUG MARKER: /usr/sbin/lctl mark FID seq 0x200000400, oid 0x67ff ver 0x0 Lustre: DEBUG MARKER: FID seq 0x200000400, oid 0x67ff ver 0x0 Lustre: DEBUG MARKER: /usr/sbin/lctl mark LOV seq 0x200000400, oid 0x67ff, count: 2 Lustre: DEBUG MARKER: LOV seq 0x200000400, oid 0x67ff, count: 2 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27A: check filesystem-wide default LOV EA values == 22:26:10 $1414387570$ Lustre: DEBUG MARKER: == sanity test 27A: check filesystem-wide default LOV EA values == 22:26:10 (1414387570) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27B: call setstripe on open unlinked file\/rename victim == 22:26:11 $1414387571$ Lustre: DEBUG MARKER: == sanity test 27B: call setstripe on open unlinked file/rename victim == 22:26:11 (1414387571) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 27C: check full striping across all OSTs == 22:26:13 $1414387573$ Lustre: DEBUG MARKER: == sanity test 27C: check full striping across all OSTs == 22:26:13 (1414387573) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 28: create\/mknod\/mkdir with bad file types ============== 22:26:14 $1414387574$ Lustre: DEBUG MARKER: == sanity test 28: create/mknod/mkdir with bad file types ============== 22:26:14 (1414387574) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 29: IT_GETATTR regression ============================== 22:26:16 $1414387576$ Lustre: DEBUG MARKER: == sanity test 29: IT_GETATTR regression ============================== 22:26:16 (1414387576) Lustre: DEBUG MARKER: /usr/sbin/lctl mark first d29 Lustre: DEBUG MARKER: first d29 Lustre: DEBUG MARKER: /usr/sbin/lctl mark second d29 Lustre: DEBUG MARKER: second d29 Lustre: DEBUG MARKER: /usr/sbin/lctl mark done Lustre: DEBUG MARKER: done Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 30a: execute binary from Lustre $execve$ ================ 22:26:18 $1414387578$ Lustre: DEBUG MARKER: == sanity test 30a: execute binary from Lustre (execve) ================ 22:26:18 (1414387578) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 30b: execute binary from Lustre as non-root ============= 22:26:19 $1414387579$ Lustre: DEBUG MARKER: == sanity test 30b: execute binary from Lustre as non-root ============= 22:26:19 (1414387579) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 30c: execute binary from Lustre without read perms ====== 22:26:21 $1414387581$ Lustre: DEBUG MARKER: == sanity test 30c: execute binary from Lustre without read perms ====== 22:26:21 (1414387581) LustreError: 8999:0:(ost_handler.c:1775:ost_blocking_ast()) Error -2 syncing data on lock cancel Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 31a: open-unlink file ==================================== 22:26:25 $1414387585$ Lustre: DEBUG MARKER: == sanity test 31a: open-unlink file ==================================== 22:26:25 (1414387585) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 31b: unlink file with multiple links while open ========= 22:26:26 $1414387586$ Lustre: DEBUG MARKER: == sanity test 31b: unlink file with multiple links while open ========= 22:26:26 (1414387586) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 31c: open-unlink file with multiple links =============== 22:26:27 $1414387587$ Lustre: DEBUG MARKER: == sanity test 31c: open-unlink file with multiple links =============== 22:26:27 (1414387587) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 31d: remove of open directory =========================== 22:26:29 $1414387589$ Lustre: DEBUG MARKER: == sanity test 31d: remove of open directory =========================== 22:26:29 (1414387589) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 31e: remove of open non-empty directory ================= 22:26:30 $1414387590$ Lustre: DEBUG MARKER: == sanity test 31e: remove of open non-empty directory ================= 22:26:30 (1414387590) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 31f: remove of open directory with open-unlink file ===== 22:26:31 $1414387591$ Lustre: DEBUG MARKER: == sanity test 31f: remove of open directory with open-unlink file ===== 22:26:31 (1414387591) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 31g: cross directory link================= 22:26:38 $1414387598$ Lustre: DEBUG MARKER: == sanity test 31g: cross directory link================= 22:26:38 (1414387598) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 31h: cross directory link under child================= 22:26:40 $1414387600$ Lustre: DEBUG MARKER: == sanity test 31h: cross directory link under child================= 22:26:40 (1414387600) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 31i: cross directory link under parent================= 22:26:41 $1414387601$ Lustre: DEBUG MARKER: == sanity test 31i: cross directory link under parent================= 22:26:41 (1414387601) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 31j: link for directory================= 22:26:42 $1414387602$ Lustre: DEBUG MARKER: == sanity test 31j: link for directory================= 22:26:42 (1414387602) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 31k: link to file: the same, non-existing, dir================= 22:26:44 $1414387604$ Lustre: DEBUG MARKER: == sanity test 31k: link to file: the same, non-existing, dir================= 22:26:44 (1414387604) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 31m: link to file: the same, non-existing, dir================= 22:26:45 $1414387605$ Lustre: DEBUG MARKER: == sanity test 31m: link to file: the same, non-existing, dir================= 22:26:45 (1414387605) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 31n: check link count of unlinked file == 22:26:46 $1414387606$ Lustre: DEBUG MARKER: == sanity test 31n: check link count of unlinked file == 22:26:46 (1414387606) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 31o: duplicate hard links with same filename == 22:26:47 $1414387607$ Lustre: DEBUG MARKER: == sanity test 31o: duplicate hard links with same filename == 22:26:47 (1414387607) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32a: stat d32a\/ext2-mountpoint\/.. ======================= 22:27:07 $1414387627$ Lustre: DEBUG MARKER: == sanity test 32a: stat d32a/ext2-mountpoint/.. ======================= 22:27:07 (1414387627) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32b: open d32b\/ext2-mountpoint\/.. ======================= 22:27:08 $1414387628$ Lustre: DEBUG MARKER: == sanity test 32b: open d32b/ext2-mountpoint/.. ======================= 22:27:08 (1414387628) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32c: stat d32c\/ext2-mountpoint\/..\/d2\/test_dir =========== 22:27:10 $1414387630$ Lustre: DEBUG MARKER: == sanity test 32c: stat d32c/ext2-mountpoint/../d2/test_dir =========== 22:27:10 (1414387630) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32d: open d32d\/ext2-mountpoint\/..\/d2\/test_dir =========== 22:27:11 $1414387631$ Lustre: DEBUG MARKER: == sanity test 32d: open d32d/ext2-mountpoint/../d2/test_dir =========== 22:27:11 (1414387631) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32e: stat d32e\/symlink-\>tmp\/symlink-\>lustre-subdir ====== 22:27:12 $1414387632$ Lustre: DEBUG MARKER: == sanity test 32e: stat d32e/symlink->tmp/symlink->lustre-subdir ====== 22:27:12 (1414387632) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32f: open d32f\/symlink-\>tmp\/symlink-\>lustre-subdir ====== 22:27:14 $1414387634$ Lustre: DEBUG MARKER: == sanity test 32f: open d32f/symlink->tmp/symlink->lustre-subdir ====== 22:27:14 (1414387634) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32g: stat d32g\/symlink-\>tmp\/symlink-\>lustre-subdir\/2 == 22:27:15 $1414387635$ Lustre: DEBUG MARKER: == sanity test 32g: stat d32g/symlink->tmp/symlink->lustre-subdir/2 == 22:27:15 (1414387635) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32h: open d32h\/symlink-\>tmp\/symlink-\>lustre-subdir\/2 == 22:27:16 $1414387636$ Lustre: DEBUG MARKER: == sanity test 32h: open d32h/symlink->tmp/symlink->lustre-subdir/2 == 22:27:16 (1414387636) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32i: stat d32i\/ext2-mountpoint\/..\/test_file ============= 22:27:18 $1414387638$ Lustre: DEBUG MARKER: == sanity test 32i: stat d32i/ext2-mountpoint/../test_file ============= 22:27:18 (1414387638) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32j: open d32j\/ext2-mountpoint\/..\/test_file ============= 22:27:19 $1414387639$ Lustre: DEBUG MARKER: == sanity test 32j: open d32j/ext2-mountpoint/../test_file ============= 22:27:19 (1414387639) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32k: stat d32k\/ext2-mountpoint\/..\/d2\/test_file ========== 22:27:20 $1414387640$ Lustre: DEBUG MARKER: == sanity test 32k: stat d32k/ext2-mountpoint/../d2/test_file ========== 22:27:20 (1414387640) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32l: open d32l\/ext2-mountpoint\/..\/d2\/test_file ========== 22:27:22 $1414387642$ Lustre: DEBUG MARKER: == sanity test 32l: open d32l/ext2-mountpoint/../d2/test_file ========== 22:27:22 (1414387642) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32m: stat d32m\/symlink-\>tmp\/symlink-\>lustre-root ======== 22:27:23 $1414387643$ Lustre: DEBUG MARKER: == sanity test 32m: stat d32m/symlink->tmp/symlink->lustre-root ======== 22:27:23 (1414387643) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32n: open d32n\/symlink-\>tmp\/symlink-\>lustre-root ======== 22:27:24 $1414387644$ Lustre: DEBUG MARKER: == sanity test 32n: open d32n/symlink->tmp/symlink->lustre-root ======== 22:27:24 (1414387644) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32o: stat d32o\/symlink-\>tmp\/symlink-\>lustre-root\/ == 22:27:26 $1414387646$ Lustre: DEBUG MARKER: == sanity test 32o: stat d32o/symlink->tmp/symlink->lustre-root/ == 22:27:26 (1414387646) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32p: open d32p\/symlink-\>tmp\/symlink-\>lustre-root\/ == 22:27:27 $1414387647$ Lustre: DEBUG MARKER: == sanity test 32p: open d32p/symlink->tmp/symlink->lustre-root/ == 22:27:27 (1414387647) Lustre: DEBUG MARKER: /usr/sbin/lctl mark 32p_1 Lustre: DEBUG MARKER: 32p_1 Lustre: DEBUG MARKER: /usr/sbin/lctl mark 32p_2 Lustre: DEBUG MARKER: 32p_2 Lustre: DEBUG MARKER: /usr/sbin/lctl mark 32p_3 Lustre: DEBUG MARKER: 32p_3 Lustre: DEBUG MARKER: /usr/sbin/lctl mark 32p_4 Lustre: DEBUG MARKER: 32p_4 Lustre: DEBUG MARKER: /usr/sbin/lctl mark 32p_5 Lustre: DEBUG MARKER: 32p_5 Lustre: DEBUG MARKER: /usr/sbin/lctl mark 32p_6 Lustre: DEBUG MARKER: 32p_6 Lustre: DEBUG MARKER: /usr/sbin/lctl mark 32p_7 Lustre: DEBUG MARKER: 32p_7 Lustre: DEBUG MARKER: /usr/sbin/lctl mark 32p_8 Lustre: DEBUG MARKER: 32p_8 Lustre: DEBUG MARKER: /usr/sbin/lctl mark 32p_9 Lustre: DEBUG MARKER: 32p_9 Lustre: DEBUG MARKER: /usr/sbin/lctl mark 32p_10 Lustre: DEBUG MARKER: 32p_10 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32q: stat follows mountpoints in Lustre $should return error$ == 22:27:33 $1414387653$ Lustre: DEBUG MARKER: == sanity test 32q: stat follows mountpoints in Lustre (should return error) == 22:27:33 (1414387653) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 32r: opendir follows mountpoints in Lustre $should return error$ == 22:27:34 $1414387654$ Lustre: DEBUG MARKER: == sanity test 32r: opendir follows mountpoints in Lustre (should return error) == 22:27:34 (1414387654) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 33aa: write file with mode 444 $should return error$ ====== 22:27:35 $1414387655$ Lustre: DEBUG MARKER: == sanity test 33aa: write file with mode 444 (should return error) ====== 22:27:35 (1414387655) Lustre: DEBUG MARKER: /usr/sbin/lctl mark 33_1 Lustre: DEBUG MARKER: 33_1 Lustre: DEBUG MARKER: /usr/sbin/lctl mark 33_2 Lustre: DEBUG MARKER: 33_2 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 33a: test open file$mode=0444$ with O_RDWR $should return error$ == 22:27:38 $1414387658$ Lustre: DEBUG MARKER: == sanity test 33a: test open file(mode=0444) with O_RDWR (should return error) == 22:27:38 (1414387658) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 33b: test open file with malformed flags $No panic and return error$ == 22:27:39 $1414387659$ Lustre: DEBUG MARKER: == sanity test 33b: test open file with malformed flags (No panic and return error) == 22:27:39 (1414387659) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 33c: test llobdstat and write_bytes == 22:27:41 $1414387661$ Lustre: DEBUG MARKER: == sanity test 33c: test llobdstat and write_bytes == 22:27:41 (1414387661) Lustre: DEBUG MARKER: lctl get_param -n obdfilter/lustre-OST0000/stats Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 33d: openfile with 444 modes and malformed flags under remote dir == 22:27:43 $1414387663$ Lustre: DEBUG MARKER: == sanity test 33d: openfile with 444 modes and malformed flags under remote dir == 22:27:43 (1414387663) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_33d needs \>= 2 MDTs Lustre: DEBUG MARKER: SKIP: sanity test_33d needs >= 2 MDTs Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 34a: truncate file that has not been opened ============= 22:27:44 $1414387664$ Lustre: DEBUG MARKER: == sanity test 34a: truncate file that has not been opened ============= 22:27:44 (1414387664) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 34b: O_RDONLY opening file doesn\'t create objects ======= 22:27:46 $1414387666$ Lustre: DEBUG MARKER: == sanity test 34b: O_RDONLY opening file doesn't create objects ======= 22:27:46 (1414387666) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 34c: O_RDWR opening file-with-size works ================ 22:27:47 $1414387667$ Lustre: DEBUG MARKER: == sanity test 34c: O_RDWR opening file-with-size works ================ 22:27:47 (1414387667) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 34d: write to sparse file =============================== 22:27:48 $1414387668$ Lustre: DEBUG MARKER: == sanity test 34d: write to sparse file =============================== 22:27:48 (1414387668) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 34e: create objects, some with size and some without ==== 22:27:49 $1414387669$ Lustre: DEBUG MARKER: == sanity test 34e: create objects, some with size and some without ==== 22:27:49 (1414387669) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 34f: read from a file with no objects until EOF ========= 22:27:51 $1414387671$ Lustre: DEBUG MARKER: == sanity test 34f: read from a file with no objects until EOF ========= 22:27:51 (1414387671) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 34g: truncate long file ================================= 22:27:52 $1414387672$ Lustre: DEBUG MARKER: == sanity test 34g: truncate long file ================================= 22:27:52 (1414387672) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_34h skipping ALWAYS excluded test 34h Lustre: DEBUG MARKER: SKIP: sanity test_34h skipping ALWAYS excluded test 34h Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 35a: exec file with mode 444 $should return and not leak$ ======= 22:27:57 $1414387677$ Lustre: DEBUG MARKER: == sanity test 35a: exec file with mode 444 (should return and not leak) ======= 22:27:57 (1414387677) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 36a: MDS utime check $mknod, utime$ ===================== 22:27:58 $1414387678$ Lustre: DEBUG MARKER: == sanity test 36a: MDS utime check (mknod, utime) ===================== 22:27:58 (1414387678) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 36b: OST utime check $open, utime$ ====================== 22:27:59 $1414387679$ Lustre: DEBUG MARKER: == sanity test 36b: OST utime check (open, utime) ====================== 22:27:59 (1414387679) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 36c: non-root MDS utime check $mknod, utime$ ============ 22:28:00 $1414387680$ Lustre: DEBUG MARKER: == sanity test 36c: non-root MDS utime check (mknod, utime) ============ 22:28:00 (1414387680) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 36d: non-root OST utime check $open, utime$ ============= 22:28:02 $1414387682$ Lustre: DEBUG MARKER: == sanity test 36d: non-root OST utime check (open, utime) ============= 22:28:02 (1414387682) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 36e: utime on non-owned file $should return error$ ====== 22:28:03 $1414387683$ Lustre: DEBUG MARKER: == sanity test 36e: utime on non-owned file (should return error) ====== 22:28:03 (1414387683) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 36f: utime on file racing with OST BRW write ============ 22:28:05 $1414387685$ Lustre: DEBUG MARKER: == sanity test 36f: utime on file racing with OST BRW write ============ 22:28:05 (1414387685) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 36g: filter mod data cache expiry ======================= 22:28:09 $1414387689$ Lustre: DEBUG MARKER: == sanity test 36g: filter mod data cache expiry ======================= 22:28:09 (1414387689) Lustre: DEBUG MARKER: lctl get_param -n obdfilter.*.client_cache_seconds 2> /dev/null | head -n 1 Lustre: DEBUG MARKER: awk '/ll_fmd_cache/ {print $2}' /proc/slabinfo Lustre: DEBUG MARKER: awk '/ll_fmd_cache/ {print $2}' /proc/slabinfo Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 36h: utime on file racing with OST BRW write ============ 22:30:13 $1414387813$ Lustre: DEBUG MARKER: == sanity test 36h: utime on file racing with OST BRW write ============ 22:30:13 (1414387813) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 38: open a regular file with O_DIRECTORY should return -ENOTDIR ===== 22:30:16 $1414387816$ Lustre: DEBUG MARKER: == sanity test 38: open a regular file with O_DIRECTORY should return -ENOTDIR ===== 22:30:16 (1414387816) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 39: mtime changed on create ============================= 22:30:18 $1414387818$ Lustre: DEBUG MARKER: == sanity test 39: mtime changed on create ============================= 22:30:18 (1414387818) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 39b: mtime change on open, link, unlink, rename ======== 22:30:21 $1414387821$ Lustre: DEBUG MARKER: == sanity test 39b: mtime change on open, link, unlink, rename ======== 22:30:21 (1414387821) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 39c: mtime change on rename ============================= 22:30:26 $1414387826$ Lustre: DEBUG MARKER: == sanity test 39c: mtime change on rename ============================= 22:30:26 (1414387826) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 39d: create, utime, stat ================================ 22:30:31 $1414387831$ Lustre: DEBUG MARKER: == sanity test 39d: create, utime, stat ================================ 22:30:31 (1414387831) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 39e: create, stat, utime, stat ========================== 22:30:33 $1414387833$ Lustre: DEBUG MARKER: == sanity test 39e: create, stat, utime, stat ========================== 22:30:33 (1414387833) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 39f: create, stat, sleep, utime, stat =================== 22:30:34 $1414387834$ Lustre: DEBUG MARKER: == sanity test 39f: create, stat, sleep, utime, stat =================== 22:30:34 (1414387834) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 39g: write, chmod, stat ================================= 22:30:37 $1414387837$ Lustre: DEBUG MARKER: == sanity test 39g: write, chmod, stat ================================= 22:30:37 (1414387837) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 39h: write, utime within one second, stat =============== 22:30:42 $1414387842$ Lustre: DEBUG MARKER: == sanity test 39h: write, utime within one second, stat =============== 22:30:42 (1414387842) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 39i: write, rename, stat ================================ 22:30:45 $1414387845$ Lustre: DEBUG MARKER: == sanity test 39i: write, rename, stat ================================ 22:30:45 (1414387845) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 39j: write, rename, close, stat ========================= 22:30:49 $1414387849$ Lustre: DEBUG MARKER: == sanity test 39j: write, rename, close, stat ========================= 22:30:49 (1414387849) Lustre: DEBUG MARKER: /usr/sbin/lctl set_param debug_mb=150 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param debug=-1; Lustre: DEBUG MARKER: /usr/sbin/lctl set_param debug_mb=2 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param debug="trace inode super ext2 malloc cache info ioctl neterror net warning buffs other dentry nettrace page dlmtrace error emerg ha rpctrace vfstrace reada mmap config console quota sec lfsck hsm"; Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 39k: write, utime, close, stat ========================== 22:30:55 $1414387855$ Lustre: DEBUG MARKER: == sanity test 39k: write, utime, close, stat ========================== 22:30:55 (1414387855) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 39l: directory atime update ============================= 22:31:00 $1414387860$ Lustre: DEBUG MARKER: == sanity test 39l: directory atime update ============================= 22:31:00 (1414387860) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 39m: test atime and mtime before 1970 == 22:31:08 $1414387868$ Lustre: DEBUG MARKER: == sanity test 39m: test atime and mtime before 1970 == 22:31:08 (1414387868) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 39n: check that O_NOATIME is honored == 22:31:11 $1414387871$ Lustre: DEBUG MARKER: == sanity test 39n: check that O_NOATIME is honored == 22:31:11 (1414387871) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 39o: directory cached attributes updated after create ========== 22:31:25 $1414387885$ Lustre: DEBUG MARKER: == sanity test 39o: directory cached attributes updated after create ========== 22:31:25 (1414387885) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_40 skipping ALWAYS excluded test 40 Lustre: DEBUG MARKER: SKIP: sanity test_40 skipping ALWAYS excluded test 40 Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 41: test small file write + fstat ======================= 22:31:26 $1414387886$ Lustre: DEBUG MARKER: == sanity test 41: test small file write + fstat ======================= 22:31:26 (1414387886) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_42a skipping ALWAYS excluded test 42a Lustre: DEBUG MARKER: SKIP: sanity test_42a skipping ALWAYS excluded test 42a Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_42b skipping ALWAYS excluded test 42b Lustre: DEBUG MARKER: SKIP: sanity test_42b skipping ALWAYS excluded test 42b Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_42c skipping ALWAYS excluded test 42c Lustre: DEBUG MARKER: SKIP: sanity test_42c skipping ALWAYS excluded test 42c Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_42d skipping ALWAYS excluded test 42d Lustre: DEBUG MARKER: SKIP: sanity test_42d skipping ALWAYS excluded test 42d Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 42e: verify sub-RPC writes are not done synchronously == 22:31:30 $1414387890$ Lustre: DEBUG MARKER: == sanity test 42e: verify sub-RPC writes are not done synchronously == 22:31:30 (1414387890) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 43: execution of file opened for write should return -ETXTBSY == 22:32:53 $1414387973$ Lustre: DEBUG MARKER: == sanity test 43: execution of file opened for write should return -ETXTBSY == 22:32:53 (1414387973) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 43a: open$RDWR$ of file being executed should return -ETXTBSY == 22:32:55 $1414387975$ Lustre: DEBUG MARKER: == sanity test 43a: open(RDWR) of file being executed should return -ETXTBSY == 22:32:55 (1414387975) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 43b: truncate of file being executed should return -ETXTBSY == 22:32:57 $1414387977$ Lustre: DEBUG MARKER: == sanity test 43b: truncate of file being executed should return -ETXTBSY == 22:32:57 (1414387977) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 43c: md5sum of copy into lustre========================== 22:32:58 $1414387978$ Lustre: DEBUG MARKER: == sanity test 43c: md5sum of copy into lustre========================== 22:32:58 (1414387978) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 44: zero length read from a sparse stripe =============== 22:33:00 $1414387980$ Lustre: DEBUG MARKER: == sanity test 44: zero length read from a sparse stripe =============== 22:33:00 (1414387980) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 44a: test sparse pwrite ================================= 22:33:01 $1414387981$ Lustre: DEBUG MARKER: == sanity test 44a: test sparse pwrite ================================= 22:33:01 (1414387981) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_45 skipping ALWAYS excluded test 45 Lustre: DEBUG MARKER: SKIP: sanity test_45 skipping ALWAYS excluded test 45 Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 46: dirtying a previously written page ================== 22:33:04 $1414387984$ Lustre: DEBUG MARKER: == sanity test 46: dirtying a previously written page ================== 22:33:04 (1414387984) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_48a skipping ALWAYS excluded test 48a Lustre: DEBUG MARKER: SKIP: sanity test_48a skipping ALWAYS excluded test 48a Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 48b: Access removed working dir $should return errors$=== 22:33:06 $1414387986$ Lustre: DEBUG MARKER: == sanity test 48b: Access removed working dir (should return errors)=== 22:33:06 (1414387986) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 48c: Access removed working subdir $should return errors$ == 22:33:07 $1414387987$ Lustre: DEBUG MARKER: == sanity test 48c: Access removed working subdir (should return errors) == 22:33:07 (1414387987) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 48d: Access removed parent subdir $should return errors$ == 22:33:09 $1414387989$ Lustre: DEBUG MARKER: == sanity test 48d: Access removed parent subdir (should return errors) == 22:33:09 (1414387989) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 48e: Access to recreated parent subdir $should return errors$ == 22:33:10 $1414387990$ Lustre: DEBUG MARKER: == sanity test 48e: Access to recreated parent subdir (should return errors) == 22:33:10 (1414387990) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 49: Change max_pages_per_rpc won\'t break osc extent == 22:33:11 $1414387991$ Lustre: DEBUG MARKER: == sanity test 49: Change max_pages_per_rpc won't break osc extent == 22:33:11 (1414387991) Lustre: DEBUG MARKER: /usr/bin/lfs df Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 50: special situations: \/proc symlinks ================= 22:33:13 $1414387993$ Lustre: DEBUG MARKER: == sanity test 50: special situations: /proc symlinks ================= 22:33:13 (1414387993) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 51a: special situations: split htree with empty entry ==== 22:33:14 $1414387994$ Lustre: DEBUG MARKER: == sanity test 51a: special situations: split htree with empty entry ==== 22:33:14 (1414387994) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 51b: exceed 64k subdirectory nlink limit == 22:33:17 $1414387997$ Lustre: DEBUG MARKER: == sanity test 51b: exceed 64k subdirectory nlink limit == 22:33:17 (1414387997) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 51ba: verify nlink for many subdirectory cleanup == 22:38:39 $1414388319$ Lustre: DEBUG MARKER: == sanity test 51ba: verify nlink for many subdirectory cleanup == 22:38:39 (1414388319) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_51d skipping ALWAYS excluded test 51d Lustre: DEBUG MARKER: SKIP: sanity test_51d skipping ALWAYS excluded test 51d Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 52a: append-only flag test $should return errors$ ======= 22:57:00 $1414389420$ Lustre: DEBUG MARKER: == sanity test 52a: append-only flag test (should return errors) ======= 22:57:00 (1414389420) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 52b: immutable flag test $should return errors$ ========= 22:57:01 $1414389421$ Lustre: DEBUG MARKER: == sanity test 52b: immutable flag test (should return errors) ========= 22:57:01 (1414389421) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 53: verify that MDS and OSTs agree on pre-creation ====== 22:57:03 $1414389423$ Lustre: DEBUG MARKER: == sanity test 53: verify that MDS and OSTs agree on pre-creation ====== 22:57:03 (1414389423) Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n obdfilter.lustre-OST0000.last_id Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n obdfilter.lustre-OST0001.last_id Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 54a: unix domain socket test ============================ 22:57:07 $1414389427$ Lustre: DEBUG MARKER: == sanity test 54a: unix domain socket test ============================ 22:57:07 (1414389427) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 54b: char device works in lustre ======================== 22:57:08 $1414389428$ Lustre: DEBUG MARKER: == sanity test 54b: char device works in lustre ======================== 22:57:08 (1414389428) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 54c: block device works in lustre ======================= 22:57:09 $1414389429$ Lustre: DEBUG MARKER: == sanity test 54c: block device works in lustre ======================= 22:57:09 (1414389429) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 54d: fifo device works in lustre ======================== 22:57:13 $1414389433$ Lustre: DEBUG MARKER: == sanity test 54d: fifo device works in lustre ======================== 22:57:13 (1414389433) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 54e: console\/tty device works in lustre ======================== 22:57:14 $1414389434$ Lustre: DEBUG MARKER: == sanity test 54e: console/tty device works in lustre ======================== 22:57:14 (1414389434) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56a: check \/usr\/bin\/lfs getstripe == 22:57:15 $1414389435$ Lustre: DEBUG MARKER: == sanity test 56a: check /usr/bin/lfs getstripe == 22:57:15 (1414389435) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56g: check lfs find -name =============================== 22:57:17 $1414389437$ Lustre: DEBUG MARKER: == sanity test 56g: check lfs find -name =============================== 22:57:17 (1414389437) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56h: check lfs find ! -name =============================== 22:57:19 $1414389439$ Lustre: DEBUG MARKER: == sanity test 56h: check lfs find ! -name =============================== 22:57:19 (1414389439) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56i: check \'lfs find -ost UUID\' skips directories ========= 22:57:21 $1414389441$ Lustre: DEBUG MARKER: == sanity test 56i: check 'lfs find -ost UUID' skips directories ========= 22:57:21 (1414389441) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56j: check lfs find -type d =============================== 22:57:22 $1414389442$ Lustre: DEBUG MARKER: == sanity test 56j: check lfs find -type d =============================== 22:57:22 (1414389442) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56k: check lfs find -type f =============================== 22:57:24 $1414389444$ Lustre: DEBUG MARKER: == sanity test 56k: check lfs find -type f =============================== 22:57:24 (1414389444) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56l: check lfs find -type b =============================== 22:57:26 $1414389446$ Lustre: DEBUG MARKER: == sanity test 56l: check lfs find -type b =============================== 22:57:26 (1414389446) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56m: check lfs find -type c =============================== 22:57:28 $1414389448$ Lustre: DEBUG MARKER: == sanity test 56m: check lfs find -type c =============================== 22:57:28 (1414389448) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56n: check lfs find -type l =============================== 22:57:29 $1414389449$ Lustre: DEBUG MARKER: == sanity test 56n: check lfs find -type l =============================== 22:57:29 (1414389449) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56o: check lfs find -mtime for old files ============================ 22:57:31 $1414389451$ Lustre: DEBUG MARKER: == sanity test 56o: check lfs find -mtime for old files ============================ 22:57:31 (1414389451) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56p: check lfs find -uid and ! -uid ================================= 22:57:34 $1414389454$ Lustre: DEBUG MARKER: == sanity test 56p: check lfs find -uid and ! -uid ================================= 22:57:34 (1414389454) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56q: check lfs find -gid and ! -gid ================================= 22:57:36 $1414389456$ Lustre: DEBUG MARKER: == sanity test 56q: check lfs find -gid and ! -gid ================================= 22:57:36 (1414389456) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56r: check lfs find -size works ============================ 22:57:38 $1414389458$ Lustre: DEBUG MARKER: == sanity test 56r: check lfs find -size works ============================ 22:57:38 (1414389458) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56s: check lfs find -stripe-count works == 22:57:42 $1414389462$ Lustre: DEBUG MARKER: == sanity test 56s: check lfs find -stripe-count works == 22:57:42 (1414389462) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56t: check lfs find -stripe-size works == 22:57:45 $1414389465$ Lustre: DEBUG MARKER: == sanity test 56t: check lfs find -stripe-size works == 22:57:45 (1414389465) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56u: check lfs find -stripe-index works == 22:57:49 $1414389469$ Lustre: DEBUG MARKER: == sanity test 56u: check lfs find -stripe-index works == 22:57:49 (1414389469) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56v: check \'lfs find -mdt match with lfs getstripe -M\' ========= 22:57:52 $1414389472$ Lustre: DEBUG MARKER: == sanity test 56v: check 'lfs find -mdt match with lfs getstripe -M' ========= 22:57:52 (1414389472) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56w: check lfs_migrate -c stripe_count works == 22:57:55 $1414389475$ Lustre: DEBUG MARKER: == sanity test 56w: check lfs_migrate -c stripe_count works == 22:57:55 (1414389475) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56x: lfs migration support == 22:58:49 $1414389529$ Lustre: DEBUG MARKER: == sanity test 56x: lfs migration support == 22:58:49 (1414389529) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56y: lfs find -L raid0\|released == 22:58:57 $1414389537$ Lustre: DEBUG MARKER: == sanity test 56y: lfs find -L raid0|released == 22:58:57 (1414389537) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 56z: lfs find should continue after an error == 22:58:58 $1414389538$ Lustre: DEBUG MARKER: == sanity test 56z: lfs find should continue after an error == 22:58:58 (1414389538) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 57a: verify MDS filesystem created with large inodes ==== 22:59:01 $1414389541$ Lustre: DEBUG MARKER: == sanity test 57a: verify MDS filesystem created with large inodes ==== 22:59:01 (1414389541) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_57a Only applicable to ldiskfs-based MDTs Lustre: DEBUG MARKER: SKIP: sanity test_57a Only applicable to ldiskfs-based MDTs Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 57b: default LOV EAs are stored inside large inodes ===== 22:59:03 $1414389543$ Lustre: DEBUG MARKER: == sanity test 57b: default LOV EAs are stored inside large inodes ===== 22:59:03 (1414389543) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_57b Only applicable to ldiskfs-based MDTs Lustre: DEBUG MARKER: SKIP: sanity test_57b Only applicable to ldiskfs-based MDTs Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 58: verify cross-platform wire constants ================ 22:59:05 $1414389545$ Lustre: DEBUG MARKER: == sanity test 58: verify cross-platform wire constants ================ 22:59:05 (1414389545) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 59: verify cancellation of llog records async =========== 22:59:06 $1414389546$ Lustre: DEBUG MARKER: == sanity test 59: verify cancellation of llog records async =========== 22:59:06 (1414389546) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 60a: llog sanity tests run from kernel module ============ 22:59:20 $1414389560$ Lustre: DEBUG MARKER: == sanity test 60a: llog sanity tests run from kernel module ============ 22:59:20 (1414389560) Lustre: DEBUG MARKER: /usr/sbin/lctl mark test_60 run 10610 - from kernel mode Lustre: DEBUG MARKER: test_60 run 10610 - from kernel mode Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 60b: limit repeated messages from CERROR\/CWARN ========== 23:00:52 $1414389652$ Lustre: DEBUG MARKER: == sanity test 60b: limit repeated messages from CERROR/CWARN ========== 23:00:52 (1414389652) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 60c: unlink file when mds full == 23:00:53 $1414389653$ Lustre: DEBUG MARKER: == sanity test 60c: unlink file when mds full == 23:00:53 (1414389653) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 60d: test printk console message masking == 23:02:40 $1414389760$ Lustre: DEBUG MARKER: == sanity test 60d: test printk console message masking == 23:02:40 (1414389760) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 61: mmap writes don\'t make sync hang ================== 23:02:42 $1414389762$ Lustre: DEBUG MARKER: == sanity test 61: mmap() writes don't make sync hang ================== 23:02:42 (1414389762) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 63a: Verify oig_wait interruption does not crash ========= 23:02:45 $1414389765$ Lustre: DEBUG MARKER: == sanity test 63a: Verify oig_wait interruption does not crash ========= 23:02:45 (1414389765) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 63b: async write errors should be returned to fsync ===== 23:03:47 $1414389827$ Lustre: DEBUG MARKER: == sanity test 63b: async write errors should be returned to fsync ===== 23:03:47 (1414389827) Lustre: DEBUG MARKER: /usr/sbin/lctl set_param debug="trace inode super ext2 malloc cache info ioctl neterror net warning buffs other dentry nettrace page dlmtrace error emerg ha rpctrace vfstrace reada mmap config console quota sec lfsck hsm"; Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 64a: verify filter grant calculations $in kernel$ ======= 23:03:52 $1414389832$ Lustre: DEBUG MARKER: == sanity test 64a: verify filter grant calculations (in kernel) ======= 23:03:52 (1414389832) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 64b: check out-of-space detection on client ============= 23:03:54 $1414389834$ Lustre: DEBUG MARKER: == sanity test 64b: check out-of-space detection on client ============= 23:03:54 (1414389834) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_64b oos.sh: 130500096kB free gt MAXFREE 800000kB, increase 800000 $or reduce test fs size$ to proceed Lustre: DEBUG MARKER: SKIP: sanity test_64b oos.sh: 130500096kB free gt MAXFREE 800000kB, increase 800000 (or reduce test fs size) to proceed Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 64c: verify grant shrink ========================------ == 23:03:56 $1414389836$ Lustre: DEBUG MARKER: == sanity test 64c: verify grant shrink ========================------ == 23:03:56 (1414389836) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 65a: directory with no stripe info ====================== 23:03:58 $1414389838$ Lustre: DEBUG MARKER: == sanity test 65a: directory with no stripe info ====================== 23:03:58 (1414389838) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 65b: directory setstripe -S 2097152 -i 0 -c 1 == 23:03:59 $1414389839$ Lustre: DEBUG MARKER: == sanity test 65b: directory setstripe -S 2097152 -i 0 -c 1 == 23:03:59 (1414389839) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 65c: directory setstripe -S 4194304 -i 1 -c 1 == 23:04:00 $1414389840$ Lustre: DEBUG MARKER: == sanity test 65c: directory setstripe -S 4194304 -i 1 -c 1 == 23:04:00 (1414389840) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 65d: directory setstripe -S 1048576 -c stripe_count == 23:04:02 $1414389842$ Lustre: DEBUG MARKER: == sanity test 65d: directory setstripe -S 1048576 -c stripe_count == 23:04:02 (1414389842) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 65e: directory setstripe defaults ========================= 23:04:03 $1414389843$ Lustre: DEBUG MARKER: == sanity test 65e: directory setstripe defaults ========================= 23:04:03 (1414389843) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 65f: dir setstripe permission $should return error$ ===== 23:04:05 $1414389845$ Lustre: DEBUG MARKER: == sanity test 65f: dir setstripe permission (should return error) ===== 23:04:05 (1414389845) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 65g: directory setstripe -d ============================= 23:04:06 $1414389846$ Lustre: DEBUG MARKER: == sanity test 65g: directory setstripe -d ============================= 23:04:06 (1414389846) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 65h: directory stripe info inherit ====================== 23:04:07 $1414389847$ Lustre: DEBUG MARKER: == sanity test 65h: directory stripe info inherit ====================== 23:04:07 (1414389847) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 65i: set non-default striping on root directory $bug 6367$=== 23:04:09 $1414389849$ Lustre: DEBUG MARKER: == sanity test 65i: set non-default striping on root directory (bug 6367)=== 23:04:09 (1414389849) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 65ia: getstripe on -1 default directory striping == 23:04:10 $1414389850$ Lustre: DEBUG MARKER: == sanity test 65ia: getstripe on -1 default directory striping == 23:04:10 (1414389850) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 65ib: getstripe -v on -1 default directory striping == 23:04:14 $1414389854$ Lustre: DEBUG MARKER: == sanity test 65ib: getstripe -v on -1 default directory striping == 23:04:14 (1414389854) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 65ic: new find on -1 default directory striping == 23:04:17 $1414389857$ Lustre: DEBUG MARKER: == sanity test 65ic: new find on -1 default directory striping == 23:04:17 (1414389857) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 65j: set default striping on root directory $bug 6367$=== 23:04:40 $1414389880$ Lustre: DEBUG MARKER: == sanity test 65j: set default striping on root directory (bug 6367)=== 23:04:40 (1414389880) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 65k: validate manual striping works properly with deactivated OSCs == 23:04:42 $1414389882$ Lustre: DEBUG MARKER: == sanity test 65k: validate manual striping works properly with deactivated OSCs == 23:04:42 (1414389882) Lustre: lustre-OST0000: Client lustre-MDT0000-mdtlov_UUID (at 10.100.4.147@tcp) reconnecting Lustre: lustre-OST0000: deleting orphan objects from 0x0:3961 to 0x0:4001 Lustre: lustre-OST0001: deleting orphan objects from 0x0:4887 to 0x0:4929 Lustre: lustre-OST0000: Client lustre-MDT0000-mdtlov_UUID (at 10.100.4.147@tcp) reconnecting Lustre: Skipped 1 previous similar message Lustre: lustre-OST0000: deleting orphan objects from 0x0:3961 to 0x0:4033 Lustre: lustre-OST0001: Client lustre-MDT0000-mdtlov_UUID (at 10.100.4.147@tcp) reconnecting Lustre: lustre-OST0001: deleting orphan objects from 0x0:4932 to 0x0:4961 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 65l: lfs find on -1 stripe dir ========================== 23:04:47 $1414389887$ Lustre: DEBUG MARKER: == sanity test 65l: lfs find on -1 stripe dir ========================== 23:04:47 (1414389887) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 66: update inode blocks count on client ================= 23:04:48 $1414389888$ Lustre: DEBUG MARKER: == sanity test 66: update inode blocks count on client ================= 23:04:48 (1414389888) Lustre: DEBUG MARKER: lctl set_param -n osd*.*OS*.force_sync 1 Lustre: DEBUG MARKER: lctl set_param -n osd*.*OS*.force_sync 1 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 68a: lloop driver - basic test ========================== 23:04:58 $1414389898$ Lustre: DEBUG MARKER: == sanity test 68a: lloop driver - basic test ========================== 23:04:58 (1414389898) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_68a llite_lloop module disabled Lustre: DEBUG MARKER: SKIP: sanity test_68a llite_lloop module disabled Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_68b skipping ALWAYS excluded test 68b Lustre: DEBUG MARKER: SKIP: sanity test_68b skipping ALWAYS excluded test 68b Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 69: verify oa2dentry return -ENOENT doesn\'t LBUG ======== 23:05:00 $1414389900$ Lustre: DEBUG MARKER: == sanity test 69: verify oa2dentry return -ENOENT doesn't LBUG ======== 23:05:00 (1414389900) Lustre: DEBUG MARKER: lctl set_param fail_loc=0x217 Lustre: *** cfs_fail_loc=217, val=0*** Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x217 Lustre: *** cfs_fail_loc=217, val=0*** Lustre: Skipped 7 previous similar messages Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 71: Running dbench on lustre $don\'t segment fault$ ====== 23:05:07 $1414389907$ Lustre: DEBUG MARKER: == sanity test 71: Running dbench on lustre (don't segment fault) ====== 23:05:07 (1414389907) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 72a: Test that remove suid works properly $bug5695$ ====== 23:17:15 $1414390635$ Lustre: DEBUG MARKER: == sanity test 72a: Test that remove suid works properly (bug5695) ====== 23:17:15 (1414390635) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 72b: Test that we keep mode setting if without file data changed $bug 24226$ == 23:17:19 $1414390639$ Lustre: DEBUG MARKER: == sanity test 72b: Test that we keep mode setting if without file data changed (bug 24226) == 23:17:19 (1414390639) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 73: multiple MDC requests $should not deadlock$ == 23:17:22 $1414390642$ Lustre: DEBUG MARKER: == sanity test 73: multiple MDC requests (should not deadlock) == 23:17:22 (1414390642) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 74a: ldlm_enqueue freed-export error path, ls $shouldn\'t LBUG$ == 23:17:50 $1414390670$ Lustre: DEBUG MARKER: == sanity test 74a: ldlm_enqueue freed-export error path, ls (shouldn't LBUG) == 23:17:50 (1414390670) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 74b: ldlm_enqueue freed-export error path, touch $shouldn\'t LBUG$ == 23:17:51 $1414390671$ Lustre: DEBUG MARKER: == sanity test 74b: ldlm_enqueue freed-export error path, touch (shouldn't LBUG) == 23:17:51 (1414390671) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 74c: ldlm_lock_create error path, $shouldn\'t LBUG$ == 23:17:52 $1414390672$ Lustre: DEBUG MARKER: == sanity test 74c: ldlm_lock_create error path, (shouldn't LBUG) == 23:17:52 (1414390672) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_76 skipping ALWAYS excluded test 76 Lustre: DEBUG MARKER: SKIP: sanity test_76 skipping ALWAYS excluded test 76 Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 77a: normal checksum read\/write operation == 23:17:54 $1414390674$ Lustre: DEBUG MARKER: == sanity test 77a: normal checksum read/write operation == 23:17:54 (1414390674) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 77b: checksum error on client write, read == 23:17:58 $1414390678$ Lustre: DEBUG MARKER: == sanity test 77b: checksum error on client write, read == 23:17:58 (1414390678) LustreError: 168-f: BAD WRITE CHECKSUM: lustre-OST0000 from 12345-10.100.4.141@tcp inode [0x200000400:0x1a46e:0x0] object 0x0:5281 extent [0-1048575]: client csum a441fd75, server csum a441fd74 Lustre: DEBUG MARKER: /usr/sbin/lctl mark set checksum type to crc32 Lustre: DEBUG MARKER: set checksum type to crc32 Lustre: DEBUG MARKER: /usr/sbin/lctl mark set checksum type to adler Lustre: DEBUG MARKER: set checksum type to adler Lustre: DEBUG MARKER: /usr/sbin/lctl mark set checksum type to adler Lustre: DEBUG MARKER: set checksum type to adler Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 77d: checksum error on OST direct write, read == 23:18:12 $1414390692$ Lustre: DEBUG MARKER: == sanity test 77d: checksum error on OST direct write, read == 23:18:12 (1414390692) LustreError: 168-f: BAD WRITE CHECKSUM: lustre-OST0001 from 12345-10.100.4.141@tcp inode [0x200000400:0x1a46f:0x0] object 0x0:6205 extent [0-1048575]: client csum 19eeae62, server csum 19eeae61 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 77f: repeat checksum error on write $expect error$ == 23:18:18 $1414390698$ Lustre: DEBUG MARKER: == sanity test 77f: repeat checksum error on write (expect error) == 23:18:18 (1414390698) Lustre: DEBUG MARKER: /usr/sbin/lctl mark set checksum type to crc32 Lustre: DEBUG MARKER: set checksum type to crc32 LustreError: 168-f: BAD WRITE CHECKSUM: lustre-OST0000 from 12345-10.100.4.141@tcp inode [0x200000400:0x1a471:0x0] object 0x0:5282 extent [0-1048575]: client csum 19eeae62, server csum 19eeae61 LustreError: 168-f: BAD WRITE CHECKSUM: lustre-OST0000 from 12345-10.100.4.141@tcp inode [0x200000400:0x1a471:0x0] object 0x0:5282 extent [3145728-4194303]: client csum 19eeae62, server csum 19eeae61 LustreError: Skipped 8 previous similar messages LustreError: 168-f: BAD WRITE CHECKSUM: lustre-OST0000 from 12345-10.100.4.141@tcp inode [0x200000400:0x1a471:0x0] object 0x0:5282 extent [1048576-2097151]: client csum 19eeae62, server csum 19eeae61 LustreError: Skipped 9 previous similar messages LustreError: 168-f: BAD WRITE CHECKSUM: lustre-OST0000 from 12345-10.100.4.141@tcp inode [0x200000400:0x1a471:0x0] object 0x0:5282 extent [2097152-3145727]: client csum 19eeae62, server csum 19eeae61 LustreError: Skipped 14 previous similar messages LustreError: 168-f: BAD WRITE CHECKSUM: lustre-OST0000 from 12345-10.100.4.141@tcp inode [0x200000400:0x1a471:0x0] object 0x0:5282 extent [0-1048575]: client csum 19eeae62, server csum 19eeae61 LustreError: Skipped 21 previous similar messages LustreError: 168-f: BAD WRITE CHECKSUM: lustre-OST0000 from 12345-10.100.4.141@tcp inode [0x200000400:0x1a471:0x0] object 0x0:5282 extent [2097152-3145727]: client csum 19eeae62, server csum 19eeae61 LustreError: Skipped 25 previous similar messages Lustre: DEBUG MARKER: /usr/sbin/lctl mark set checksum type to adler Lustre: DEBUG MARKER: set checksum type to adler LustreError: 168-f: BAD WRITE CHECKSUM: lustre-OST0000 from 12345-10.100.4.141@tcp inode [0x200000400:0x1a471:0x0] object 0x0:5282 extent [0-1048575]: client csum 19eeae62, server csum 19eeae61 LustreError: Skipped 86 previous similar messages Lustre: DEBUG MARKER: /usr/sbin/lctl mark set checksum type to adler Lustre: DEBUG MARKER: set checksum type to adler Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 77g: checksum error on OST write, read == 23:20:38 $1414390838$ Lustre: DEBUG MARKER: == sanity test 77g: checksum error on OST write, read == 23:20:38 (1414390838) Lustre: DEBUG MARKER: lctl set_param fail_loc=0x8000021a Lustre: *** cfs_fail_loc=21a, val=0*** Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x8000021b Lustre: *** cfs_fail_loc=21b, val=0*** Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 77i: client not supporting OSD_CONNECT_CKSUM == 23:20:47 $1414390847$ Lustre: DEBUG MARKER: == sanity test 77i: client not supporting OSD_CONNECT_CKSUM == 23:20:47 (1414390847) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 77j: client only supporting ADLER32 == 23:20:49 $1414390849$ Lustre: DEBUG MARKER: == sanity test 77j: client only supporting ADLER32 == 23:20:49 (1414390849) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 78: handle large O_DIRECT writes correctly ============== 23:20:53 $1414390853$ Lustre: DEBUG MARKER: == sanity test 78: handle large O_DIRECT writes correctly ============== 23:20:53 (1414390853) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 79: df report consistency check ========================= 23:30:21 $1414391421$ Lustre: DEBUG MARKER: == sanity test 79: df report consistency check ========================= 23:30:21 (1414391421) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 80: Page eviction is equally fast at high offsets too ====== 23:30:29 $1414391429$ Lustre: DEBUG MARKER: == sanity test 80: Page eviction is equally fast at high offsets too ====== 23:30:29 (1414391429) Lustre: DEBUG MARKER: lctl get_param -n obdfilter.*.sync_on_lock_cancel Lustre: DEBUG MARKER: lctl set_param obdfilter.*.sync_on_lock_cancel=never Lustre: DEBUG MARKER: lctl set_param obdfilter.*.sync_on_lock_cancel=always Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 81a: OST should retry write when get -ENOSPC ================= 23:30:33 $1414391433$ Lustre: DEBUG MARKER: == sanity test 81a: OST should retry write when get -ENOSPC ================= 23:30:33 (1414391433) Lustre: DEBUG MARKER: lctl set_param fail_loc=0x80000228 Lustre: *** cfs_fail_loc=228, val=0*** Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 81b: OST should return -ENOSPC when retry still fails ========= 23:30:36 $1414391436$ Lustre: DEBUG MARKER: == sanity test 81b: OST should return -ENOSPC when retry still fails ========= 23:30:36 (1414391436) Lustre: DEBUG MARKER: lctl set_param fail_loc=0x228 Lustre: *** cfs_fail_loc=228, val=0*** Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 82: Basic grouplock test ================================= 23:30:39 $1414391439$ Lustre: DEBUG MARKER: == sanity test 82: Basic grouplock test ================================= 23:30:39 (1414391439) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 99a: cvs init =========================================== 23:30:45 $1414391445$ Lustre: DEBUG MARKER: == sanity test 99a: cvs init =========================================== 23:30:45 (1414391445) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 99b: cvs import ========================================= 23:30:49 $1414391449$ Lustre: DEBUG MARKER: == sanity test 99b: cvs import ========================================= 23:30:49 (1414391449) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 99c: cvs checkout ======================================= 23:30:52 $1414391452$ Lustre: DEBUG MARKER: == sanity test 99c: cvs checkout ======================================= 23:30:52 (1414391452) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 99d: cvs add ============================================ 23:31:00 $1414391460$ Lustre: DEBUG MARKER: == sanity test 99d: cvs add ============================================ 23:31:00 (1414391460) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 99e: cvs update ========================================= 23:31:02 $1414391462$ Lustre: DEBUG MARKER: == sanity test 99e: cvs update ========================================= 23:31:02 (1414391462) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 99f: cvs commit ========================================= 23:31:04 $1414391464$ Lustre: DEBUG MARKER: == sanity test 99f: cvs commit ========================================= 23:31:04 (1414391464) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 100: check local port using privileged port ============= 23:31:09 $1414391469$ Lustre: DEBUG MARKER: == sanity test 100: check local port using privileged port ============= 23:31:09 (1414391469) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 101a: check read-ahead for random reads ================== 23:31:11 $1414391471$ Lustre: DEBUG MARKER: == sanity test 101a: check read-ahead for random reads ================== 23:31:11 (1414391471) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 101b: check stride-io mode read-ahead =================== 23:32:28 $1414391548$ Lustre: DEBUG MARKER: == sanity test 101b: check stride-io mode read-ahead =================== 23:32:28 (1414391548) Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.read_cache_enable=0 osd-*.lustre-OST*.read_cache_enable=0 2>&1 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.writethrough_cache_enable=0 osd-*.lustre-OST*.writethrough_cache_enable=0 2>&1 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.read_cache_enable=1 osd-*.lustre-OST*.read_cache_enable=1 2>&1 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.writethrough_cache_enable=1 osd-*.lustre-OST*.writethrough_cache_enable=1 2>&1 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 101c: check stripe_size aligned read-ahead =================== 23:33:12 $1414391592$ Lustre: DEBUG MARKER: == sanity test 101c: check stripe_size aligned read-ahead =================== 23:33:12 (1414391592) Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.read_cache_enable=0 osd-*.lustre-OST*.read_cache_enable=0 2>&1 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.writethrough_cache_enable=0 osd-*.lustre-OST*.writethrough_cache_enable=0 2>&1 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.read_cache_enable=1 osd-*.lustre-OST*.read_cache_enable=1 2>&1 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.writethrough_cache_enable=1 osd-*.lustre-OST*.writethrough_cache_enable=1 2>&1 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 101d: file read with and without read-ahead enabled =================== 23:33:48 $1414391628$ Lustre: DEBUG MARKER: == sanity test 101d: file read with and without read-ahead enabled =================== 23:33:48 (1414391628) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 101e: check read-ahead for small read$1k$ for small files$500k$ == 23:41:49 $1414392109$ Lustre: DEBUG MARKER: == sanity test 101e: check read-ahead for small read(1k) for small files(500k) == 23:41:49 (1414392109) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 101f: check read-ahead for max_read_ahead_whole_mb == 23:42:19 $1414392139$ Lustre: DEBUG MARKER: == sanity test 101f: check read-ahead for max_read_ahead_whole_mb == 23:42:19 (1414392139) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 102a: user xattr test ==================================== 23:42:23 $1414392143$ Lustre: DEBUG MARKER: == sanity test 102a: user xattr test ==================================== 23:42:23 (1414392143) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 102b: getfattr\/setfattr for trusted.lov EAs ============== 23:42:24 $1414392144$ Lustre: DEBUG MARKER: == sanity test 102b: getfattr/setfattr for trusted.lov EAs ============== 23:42:24 (1414392144) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 102c: non-root getfattr\/setfattr for lustre.lov EAs ============= 23:42:25 $1414392145$ Lustre: DEBUG MARKER: == sanity test 102c: non-root getfattr/setfattr for lustre.lov EAs ============= 23:42:25 (1414392145) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 102d: tar restore stripe info from tarfile,not keep osts ============= 23:42:28 $1414392148$ Lustre: DEBUG MARKER: == sanity test 102d: tar restore stripe info from tarfile,not keep osts ============= 23:42:28 (1414392148) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 102f: tar copy files, not keep osts ============= 23:42:33 $1414392153$ Lustre: DEBUG MARKER: == sanity test 102f: tar copy files, not keep osts ============= 23:42:33 (1414392153) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 102h: grow xattr from inside inode to external block == 23:42:38 $1414392158$ Lustre: DEBUG MARKER: == sanity test 102h: grow xattr from inside inode to external block == 23:42:38 (1414392158) Lustre: DEBUG MARKER: /usr/sbin/lctl mark save trusted.big on \/mnt\/lustre\/f102h.sanity Lustre: DEBUG MARKER: save trusted.big on /mnt/lustre/f102h.sanity Lustre: DEBUG MARKER: /usr/sbin/lctl mark save trusted.sml on \/mnt\/lustre\/f102h.sanity Lustre: DEBUG MARKER: save trusted.sml on /mnt/lustre/f102h.sanity Lustre: DEBUG MARKER: /usr/sbin/lctl mark grow trusted.sml on \/mnt\/lustre\/f102h.sanity Lustre: DEBUG MARKER: grow trusted.sml on /mnt/lustre/f102h.sanity Lustre: DEBUG MARKER: /usr/sbin/lctl mark trusted.big still valid after growing trusted.sml Lustre: DEBUG MARKER: trusted.big still valid after growing trusted.sml Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 102ha: grow xattr from inside inode to external inode == 23:42:41 $1414392161$ Lustre: DEBUG MARKER: == sanity test 102ha: grow xattr from inside inode to external inode == 23:42:41 (1414392161) Lustre: DEBUG MARKER: /usr/sbin/lctl mark save trusted.big on \/mnt\/lustre\/f102ha.sanity Lustre: DEBUG MARKER: save trusted.big on /mnt/lustre/f102ha.sanity Lustre: DEBUG MARKER: /usr/sbin/lctl mark save trusted.sml on \/mnt\/lustre\/f102ha.sanity Lustre: DEBUG MARKER: save trusted.sml on /mnt/lustre/f102ha.sanity Lustre: DEBUG MARKER: /usr/sbin/lctl mark grow trusted.sml on \/mnt\/lustre\/f102ha.sanity Lustre: DEBUG MARKER: grow trusted.sml on /mnt/lustre/f102ha.sanity Lustre: DEBUG MARKER: /usr/sbin/lctl mark trusted.big still valid after growing trusted.sml Lustre: DEBUG MARKER: trusted.big still valid after growing trusted.sml Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 102i: lgetxattr test on symbolic link ============== 23:42:45 $1414392165$ Lustre: DEBUG MARKER: == sanity test 102i: lgetxattr test on symbolic link ============== 23:42:45 (1414392165) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 102j: non-root tar restore stripe info from tarfile, not keep osts ===== 23:42:46 $1414392166$ Lustre: DEBUG MARKER: == sanity test 102j: non-root tar restore stripe info from tarfile, not keep osts ===== 23:42:46 (1414392166) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 102k: setfattr without parameter of value shouldn\'t cause a crash == 23:42:51 $1414392171$ Lustre: DEBUG MARKER: == sanity test 102k: setfattr without parameter of value shouldn't cause a crash == 23:42:51 (1414392171) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 102l: listxattr size test ==================================== 23:42:53 $1414392173$ Lustre: DEBUG MARKER: == sanity test 102l: listxattr size test ==================================== 23:42:53 (1414392173) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 102m: Ensure listxattr fails on small bufffer ========== 23:42:54 $1414392174$ Lustre: DEBUG MARKER: == sanity test 102m: Ensure listxattr fails on small bufffer ========== 23:42:54 (1414392174) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 102n: silently ignore setxattr on internal trusted xattrs == 23:42:55 $1414392175$ Lustre: DEBUG MARKER: == sanity test 102n: silently ignore setxattr on internal trusted xattrs == 23:42:55 (1414392175) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_102n MDT \< 2.5.50 allows setxattr on internal trusted xattrs Lustre: DEBUG MARKER: SKIP: sanity test_102n MDT < 2.5.50 allows setxattr on internal trusted xattrs Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 103a: acl test =========================================== 23:42:57 $1414392177$ Lustre: DEBUG MARKER: == sanity test 103a: acl test =========================================== 23:42:57 (1414392177) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 103b: MDS mount option \'noacl\' == 23:43:35 $1414392215$ Lustre: DEBUG MARKER: == sanity test 103b: MDS mount option 'noacl' == 23:43:35 (1414392215) LustreError: 11-0: lustre-MDT0000-lwp-OST0000: Communicating with 10.100.4.147@tcp, operation obd_ping failed with -107. Lustre: lustre-MDT0000-lwp-OST0000: Connection to lustre-MDT0000 (at 10.100.4.147@tcp) was lost; in progress operations using this service will wait for recovery to complete Lustre: Skipped 1 previous similar message Lustre: 8986:0:(client.c:1918:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1414392218/real 1414392218] req@ffff880045ae1000 x1483091919007108/t0(0) o400->MGC10.100.4.147@tcp@10.100.4.147@tcp:26/25 lens 224/224 e 0 to 1 dl 1414392225 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 Lustre: 8986:0:(client.c:1918:ptlrpc_expire_one_request()) Skipped 1 previous similar message LustreError: 166-1: MGC10.100.4.147@tcp: Connection to MGS (at 10.100.4.147@tcp) was lost; in progress operations using this service will fail Lustre: 8984:0:(client.c:1918:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1414392225/real 1414392225] req@ffff88004dd00400 x1483091919007132/t0(0) o250->MGC10.100.4.147@tcp@10.100.4.147@tcp:26/25 lens 400/544 e 0 to 1 dl 1414392231 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 Lustre: lustre-OST0000: deleting orphan objects from 0x0:6021 to 0x0:6049 Lustre: lustre-OST0001: deleting orphan objects from 0x0:6940 to 0x0:6977 Lustre: Evicted from MGS (at 10.100.4.147@tcp) after server handle changed from 0xe51f1a4c383eea83 to 0xe51f1a4c38d0d768 Lustre: MGC10.100.4.147@tcp: Connection restored to MGS (at 10.100.4.147@tcp) Lustre: Skipped 1 previous similar message Lustre: 8984:0:(client.c:1918:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1414392225/real 1414392225] req@ffff88004dc39c00 x1483091919007136/t0(0) o38->lustre-MDT0000-lwp-OST0000@10.100.4.147@tcp:12/10 lens 400/544 e 0 to 1 dl 1414392236 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 LustreError: 11-0: MGC10.100.4.147@tcp: Communicating with 10.100.4.147@tcp, operation obd_ping failed with -107. LustreError: Skipped 1 previous similar message LustreError: 166-1: MGC10.100.4.147@tcp: Connection to MGS (at 10.100.4.147@tcp) was lost; in progress operations using this service will fail Lustre: Evicted from MGS (at 10.100.4.147@tcp) after server handle changed from 0xe51f1a4c38d0d768 to 0xe51f1a4c38d0d7bc Lustre: MGC10.100.4.147@tcp: Connection restored to MGS (at 10.100.4.147@tcp) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 103c: \'cp -rp\' won\'t set empty acl == 23:44:14 $1414392254$ Lustre: DEBUG MARKER: == sanity test 103c: 'cp -rp' won't set empty acl == 23:44:14 (1414392254) Lustre: 8984:0:(client.c:1918:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1414392245/real 1414392245] req@ffff88004dc39c00 x1483091919007164/t0(0) o38->lustre-MDT0000-lwp-OST0000@10.100.4.147@tcp:12/10 lens 400/544 e 0 to 1 dl 1414392266 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 Lustre: 8984:0:(client.c:1918:ptlrpc_expire_one_request()) Skipped 1 previous similar message LustreError: 167-0: lustre-MDT0000-lwp-OST0000: This client was evicted by lustre-MDT0000; in progress operations using this service will fail. LustreError: Skipped 1 previous similar message Lustre: lustre-MDT0000-lwp-OST0000: Connection restored to lustre-MDT0000 (at 10.100.4.147@tcp) Lustre: lustre-OST0000: deleting orphan objects from 0x0:6051 to 0x0:6081 Lustre: lustre-OST0001: deleting orphan objects from 0x0:6940 to 0x0:7009 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 104a: lfs df [-ih] [path] test =========================== 23:44:42 $1414392282$ Lustre: DEBUG MARKER: == sanity test 104a: lfs df [-ih] [path] test =========================== 23:44:42 (1414392282) Lustre: lustre-OST0000: Client 5a424c15-5e79-79f5-cc4b-64281f651459 (at 10.100.4.141@tcp) reconnecting Lustre: DEBUG MARKER: /usr/sbin/lctl mark osc.lustre-OST0000-osc-ffff880069288c00.ost_server_uuid in FULL state after 0 sec Lustre: DEBUG MARKER: osc.lustre-OST0000-osc-ffff880069288c00.ost_server_uuid in FULL state after 0 sec Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 104b: runas -u 500 -g 500 lfs check servers test ====================== 23:44:44 $1414392284$ Lustre: DEBUG MARKER: == sanity test 104b: runas -u 500 -g 500 lfs check servers test ====================== 23:44:44 (1414392284) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 105a: flock when mounted without -o flock test ========== 23:44:45 $1414392285$ Lustre: DEBUG MARKER: == sanity test 105a: flock when mounted without -o flock test ========== 23:44:45 (1414392285) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 105b: fcntl when mounted without -o flock test ========== 23:44:46 $1414392286$ Lustre: DEBUG MARKER: == sanity test 105b: fcntl when mounted without -o flock test ========== 23:44:46 (1414392286) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 105c: lockf when mounted without -o flock test ========== 23:44:48 $1414392288$ Lustre: DEBUG MARKER: == sanity test 105c: lockf when mounted without -o flock test ========== 23:44:48 (1414392288) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 105d: flock race $should not freeze$ ========== 23:44:49 $1414392289$ Lustre: DEBUG MARKER: == sanity test 105d: flock race (should not freeze) ========== 23:44:49 (1414392289) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 105e: Two conflicting flocks from same process ========= 23:45:00 $1414392300$ Lustre: DEBUG MARKER: == sanity test 105e: Two conflicting flocks from same process ========= 23:45:00 (1414392300) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 106: attempt exec of dir followed by chown of that dir == 23:45:02 $1414392302$ Lustre: DEBUG MARKER: == sanity test 106: attempt exec of dir followed by chown of that dir == 23:45:02 (1414392302) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 107: Coredump on SIG == 23:45:03 $1414392303$ Lustre: DEBUG MARKER: == sanity test 107: Coredump on SIG == 23:45:03 (1414392303) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 110: filename length checking == 23:45:05 $1414392305$ Lustre: DEBUG MARKER: == sanity test 110: filename length checking == 23:45:05 (1414392305) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 115: verify dynamic thread creation====================== 23:45:07 $1414392307$ Lustre: DEBUG MARKER: == sanity test 115: verify dynamic thread creation====================== 23:45:07 (1414392307) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_115 no OSS threads Lustre: DEBUG MARKER: SKIP: sanity test_115 no OSS threads Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 116a: stripe QOS: free space balance ===================== 23:45:09 $1414392309$ Lustre: DEBUG MARKER: == sanity test 116a: stripe QOS: free space balance ===================== 23:45:09 (1414392309) format at ldlm_pool.c:628:ldlm_pool_recalc doesn't end in newline Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 116b: QoS shouldn\'t LBUG if not enough OSTs found on the 2nd pass == 23:57:19 $1414393039$ Lustre: DEBUG MARKER: == sanity test 116b: QoS shouldn't LBUG if not enough OSTs found on the 2nd pass == 23:57:19 (1414393039) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_116b no QOS Lustre: DEBUG MARKER: SKIP: sanity test_116b no QOS Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 117: verify fsfilt_extend ============ 23:57:20 $1414393040$ Lustre: DEBUG MARKER: == sanity test 117: verify fsfilt_extend ============ 23:57:20 (1414393040) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 118a: verify O_SYNC works ============ 23:57:22 $1414393042$ Lustre: DEBUG MARKER: == sanity test 118a: verify O_SYNC works ============ 23:57:22 (1414393042) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 118b: Reclaim dirty pages on fatal error ============ 23:57:27 $1414393047$ Lustre: DEBUG MARKER: == sanity test 118b: Reclaim dirty pages on fatal error ============ 23:57:27 (1414393047) Lustre: DEBUG MARKER: lctl set_param fail_loc=0x217 Lustre: *** cfs_fail_loc=217, val=0*** Lustre: Skipped 3 previous similar messages Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 118c: Fsync blocks on EROFS until dirty pages are flushed ============ 23:57:33 $1414393053$ Lustre: DEBUG MARKER: == sanity test 118c: Fsync blocks on EROFS until dirty pages are flushed ============ 23:57:33 (1414393053) Lustre: DEBUG MARKER: lctl set_param fail_loc=0x216 LustreError: 9939:0:(fail.c:133:__cfs_fail_timeout_set()) cfs_fail_timeout id 216 sleeping for 1000ms LustreError: 9939:0:(fail.c:137:__cfs_fail_timeout_set()) cfs_fail_timeout id 216 awake Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 118d: Fsync validation inject a delay of the bulk ============ 23:57:38 $1414393058$ Lustre: DEBUG MARKER: == sanity test 118d: Fsync validation inject a delay of the bulk ============ 23:57:38 (1414393058) Lustre: DEBUG MARKER: lctl set_param fail_loc=0x214 LustreError: 9939:0:(fail.c:133:__cfs_fail_timeout_set()) cfs_fail_timeout id 214 sleeping for 5000ms LustreError: 9939:0:(fail.c:137:__cfs_fail_timeout_set()) cfs_fail_timeout id 214 awake Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 118f: Simulate unrecoverable OSC side error ============ 23:57:48 $1414393068$ Lustre: DEBUG MARKER: == sanity test 118f: Simulate unrecoverable OSC side error ============ 23:57:48 (1414393068) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 118g: Don\'t stay in wait if we got local -ENOMEM ============ 23:57:51 $1414393071$ Lustre: DEBUG MARKER: == sanity test 118g: Don't stay in wait if we got local -ENOMEM ============ 23:57:51 (1414393071) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 118h: Verify timeout in handling recoverables errors ============ 23:57:54 $1414393074$ Lustre: DEBUG MARKER: == sanity test 118h: Verify timeout in handling recoverables errors ============ 23:57:54 (1414393074) Lustre: DEBUG MARKER: lctl set_param fail_loc=0x20e Lustre: *** cfs_fail_loc=20e, val=0*** Lustre: *** cfs_fail_loc=20e, val=0*** Lustre: *** cfs_fail_loc=20e, val=0*** Lustre: *** cfs_fail_loc=20e, val=0*** Lustre: *** cfs_fail_loc=20e, val=0*** Lustre: *** cfs_fail_loc=20e, val=0*** Lustre: Skipped 1 previous similar message Lustre: *** cfs_fail_loc=20e, val=0*** Lustre: Skipped 2 previous similar messages Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 118i: Fix error before timeout in recoverable error ============ 23:58:53 $1414393133$ Lustre: DEBUG MARKER: == sanity test 118i: Fix error before timeout in recoverable error ============ 23:58:53 (1414393133) Lustre: DEBUG MARKER: lctl set_param fail_loc=0x20e Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 118j: Simulate unrecoverable OST side error ============ 23:59:04 $1414393144$ Lustre: DEBUG MARKER: == sanity test 118j: Simulate unrecoverable OST side error ============ 23:59:04 (1414393144) Lustre: DEBUG MARKER: lctl set_param fail_loc=0x220 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 118k: bio alloc -ENOMEM and IO TERM handling =========== 23:59:07 $1414393147$ Lustre: DEBUG MARKER: == sanity test 118k: bio alloc -ENOMEM and IO TERM handling =========== 23:59:07 (1414393147) Lustre: DEBUG MARKER: lctl set_param fail_loc=0x20e Lustre: *** cfs_fail_loc=20e, val=0*** Lustre: Skipped 40 previous similar messages Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 118l: fsync dir =========== 23:59:31 $1414393171$ Lustre: DEBUG MARKER: == sanity test 118l: fsync dir =========== 23:59:31 (1414393171) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 118m: fdatasync dir =========== 23:59:33 $1414393173$ Lustre: DEBUG MARKER: == sanity test 118m: fdatasync dir =========== 23:59:33 (1414393173) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 119a: Short directIO read must return actual read amount == 23:59:36 $1414393176$ Lustre: DEBUG MARKER: == sanity test 119a: Short directIO read must return actual read amount == 23:59:36 (1414393176) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 119b: Sparse directIO read must return actual read amount == 23:59:39 $1414393179$ Lustre: DEBUG MARKER: == sanity test 119b: Sparse directIO read must return actual read amount == 23:59:39 (1414393179) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 119c: Testing for direct read hitting hole == 23:59:40 $1414393180$ Lustre: DEBUG MARKER: == sanity test 119c: Testing for direct read hitting hole == 23:59:40 (1414393180) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 119d: The DIO path should try to send a new rpc once one is completed == 23:59:43 $1414393183$ Lustre: DEBUG MARKER: == sanity test 119d: The DIO path should try to send a new rpc once one is completed == 23:59:43 (1414393183) Lustre: DEBUG MARKER: /usr/sbin/lctl mark the DIO writes have completed, now wait for the reads $should not block very long$ Lustre: DEBUG MARKER: the DIO writes have completed, now wait for the reads (should not block very long) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 120a: Early Lock Cancel: mkdir test == 23:59:50 $1414393190$ Lustre: DEBUG MARKER: == sanity test 120a: Early Lock Cancel: mkdir test == 23:59:50 (1414393190) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 120b: Early Lock Cancel: create test == 23:59:51 $1414393191$ Lustre: DEBUG MARKER: == sanity test 120b: Early Lock Cancel: create test == 23:59:51 (1414393191) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 120c: Early Lock Cancel: link test == 23:59:53 $1414393193$ Lustre: DEBUG MARKER: == sanity test 120c: Early Lock Cancel: link test == 23:59:53 (1414393193) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 120d: Early Lock Cancel: setattr test == 23:59:54 $1414393194$ Lustre: DEBUG MARKER: == sanity test 120d: Early Lock Cancel: setattr test == 23:59:54 (1414393194) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 120e: Early Lock Cancel: unlink test == 23:59:56 $1414393196$ Lustre: DEBUG MARKER: == sanity test 120e: Early Lock Cancel: unlink test == 23:59:56 (1414393196) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 120f: Early Lock Cancel: rename test == 23:59:58 $1414393198$ Lustre: DEBUG MARKER: == sanity test 120f: Early Lock Cancel: rename test == 23:59:58 (1414393198) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 120g: Early Lock Cancel: performance test == 00:00:02 $1414393202$ Lustre: DEBUG MARKER: == sanity test 120g: Early Lock Cancel: performance test == 00:00:02 (1414393202) format at ldlm_pool.c:628:ldlm_pool_recalc doesn't end in newline Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 121: read cancel race =========== 00:01:43 $1414393303$ Lustre: DEBUG MARKER: == sanity test 121: read cancel race =========== 00:01:43 (1414393303) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 123a: verify statahead work == 00:01:45 $1414393305$ Lustre: DEBUG MARKER: == sanity test 123a: verify statahead work == 00:01:45 (1414393305) Lustre: DEBUG MARKER: /usr/sbin/lctl mark testing on UP system. Performance may be not as good as expected. Lustre: DEBUG MARKER: testing on UP system. Performance may be not as good as expected. Lustre: DEBUG MARKER: /usr/sbin/lctl mark ls 100 files without statahead: 1 sec Lustre: DEBUG MARKER: ls 100 files without statahead: 1 sec Lustre: DEBUG MARKER: /usr/sbin/lctl mark ls 100 files with statahead: 0 sec Lustre: DEBUG MARKER: ls 100 files with statahead: 0 sec Lustre: DEBUG MARKER: /usr/sbin/lctl mark ls 1000 files without statahead: 6 sec Lustre: DEBUG MARKER: ls 1000 files without statahead: 6 sec Lustre: DEBUG MARKER: /usr/sbin/lctl mark ls 1000 files with statahead: 5 sec Lustre: DEBUG MARKER: ls 1000 files with statahead: 5 sec Lustre: DEBUG MARKER: /usr/sbin/lctl mark ls 10000 files without statahead: 60 sec Lustre: DEBUG MARKER: ls 10000 files without statahead: 60 sec Lustre: DEBUG MARKER: /usr/sbin/lctl mark ls 10000 files with statahead: 54 sec Lustre: DEBUG MARKER: ls 10000 files with statahead: 54 sec Lustre: DEBUG MARKER: /usr/sbin/lctl mark ls done Lustre: DEBUG MARKER: ls done Lustre: DEBUG MARKER: /usr/sbin/lctl mark rm -r \/mnt\/lustre\/d123a.sanity\/: 32 seconds Lustre: DEBUG MARKER: rm -r /mnt/lustre/d123a.sanity/: 32 seconds Lustre: DEBUG MARKER: /usr/sbin/lctl mark rm done Lustre: DEBUG MARKER: rm done Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 123b: not panic with network error in statahead enqueue $bug 15027$ == 00:05:20 $1414393520$ Lustre: DEBUG MARKER: == sanity test 123b: not panic with network error in statahead enqueue (bug 15027) == 00:05:20 (1414393520) Lustre: DEBUG MARKER: /usr/sbin/lctl mark ls done Lustre: DEBUG MARKER: ls done Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 124a: lru resize ========================================= 00:05:37 $1414393537$ Lustre: DEBUG MARKER: == sanity test 124a: lru resize ========================================= 00:05:37 (1414393537) Lustre: DEBUG MARKER: /usr/sbin/lctl mark create 2000 files at \/mnt\/lustre\/d124a.sanity Lustre: DEBUG MARKER: create 2000 files at /mnt/lustre/d124a.sanity Lustre: DEBUG MARKER: /usr/sbin/lctl mark NSDIR=ldlm.namespaces.lustre-MDT0000-mdc-ffff880069288c00 Lustre: DEBUG MARKER: NSDIR=ldlm.namespaces.lustre-MDT0000-mdc-ffff880069288c00 Lustre: DEBUG MARKER: /usr/sbin/lctl mark NS=ldlm.namespaces.lustre-MDT0000-mdc-ffff880069288c00 Lustre: DEBUG MARKER: NS=ldlm.namespaces.lustre-MDT0000-mdc-ffff880069288c00 Lustre: DEBUG MARKER: /usr/sbin/lctl mark LRU=2003 Lustre: DEBUG MARKER: LRU=2003 Lustre: DEBUG MARKER: /usr/sbin/lctl mark LIMIT=93830 Lustre: DEBUG MARKER: LIMIT=93830 Lustre: DEBUG MARKER: /usr/sbin/lctl mark LVF=56213 Lustre: DEBUG MARKER: LVF=56213 Lustre: DEBUG MARKER: /usr/sbin/lctl mark OLD_LVF=1 Lustre: DEBUG MARKER: OLD_LVF=1 Lustre: DEBUG MARKER: /usr/sbin/lctl mark Sleep 50 sec Lustre: DEBUG MARKER: Sleep 50 sec Lustre: DEBUG MARKER: /usr/sbin/lctl mark Dropped 802 locks in 50s Lustre: DEBUG MARKER: Dropped 802 locks in 50s Lustre: DEBUG MARKER: /usr/sbin/lctl mark unlink 2000 files at \/mnt\/lustre\/d124a.sanity Lustre: DEBUG MARKER: unlink 2000 files at /mnt/lustre/d124a.sanity Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 124b: lru resize $performance test$ ========================= 00:07:02 $1414393622$ Lustre: DEBUG MARKER: == sanity test 124b: lru resize (performance test) ========================= 00:07:02 (1414393622) Lustre: DEBUG MARKER: /usr/sbin/lctl mark doing ls -la \/mnt\/lustre\/d124b.sanity\/disable_lru_resize 3 times Lustre: DEBUG MARKER: doing ls -la /mnt/lustre/d124b.sanity/disable_lru_resize 3 times Lustre: DEBUG MARKER: /usr/sbin/lctl mark ls -la time: 9 seconds Lustre: DEBUG MARKER: ls -la time: 9 seconds Lustre: DEBUG MARKER: /usr/sbin/lctl mark lru_size = 100 Lustre: DEBUG MARKER: lru_size = 100 Lustre: DEBUG MARKER: /usr/sbin/lctl mark doing ls -la \/mnt\/lustre\/d124b.sanity\/enable_lru_resize 3 times Lustre: DEBUG MARKER: doing ls -la /mnt/lustre/d124b.sanity/enable_lru_resize 3 times Lustre: DEBUG MARKER: /usr/sbin/lctl mark ls -la time: 9 seconds Lustre: DEBUG MARKER: ls -la time: 9 seconds Lustre: DEBUG MARKER: /usr/sbin/lctl mark lru_size = 2006 Lustre: DEBUG MARKER: lru_size = 2006 Lustre: DEBUG MARKER: /usr/sbin/lctl mark lru resize performs the same with no lru resize Lustre: DEBUG MARKER: lru resize performs the same with no lru resize Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 125: don\'t return EPROTO when a dir has a non-default striping and ACLs == 00:07:57 $1414393677$ Lustre: DEBUG MARKER: == sanity test 125: don't return EPROTO when a dir has a non-default striping and ACLs == 00:07:57 (1414393677) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 126: check that the fsgid provided by the client is taken into account == 00:08:00 $1414393680$ Lustre: DEBUG MARKER: == sanity test 126: check that the fsgid provided by the client is taken into account == 00:08:00 (1414393680) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 127a: verify the client stats are sane == 00:08:01 $1414393681$ Lustre: DEBUG MARKER: == sanity test 127a: verify the client stats are sane == 00:08:01 (1414393681) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 127b: verify the llite client stats are sane == 00:08:04 $1414393684$ Lustre: DEBUG MARKER: == sanity test 127b: verify the llite client stats are sane == 00:08:04 (1414393684) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 128: interactive lfs for 2 consecutive find\'s == 00:08:08 $1414393688$ Lustre: DEBUG MARKER: == sanity test 128: interactive lfs for 2 consecutive find's == 00:08:08 (1414393688) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 129: test directory size limit ========================== 00:08:10 $1414393690$ Lustre: DEBUG MARKER: == sanity test 129: test directory size limit ========================== 00:08:10 (1414393690) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_129 Only applicable to ldiskfs-based MDTs Lustre: DEBUG MARKER: SKIP: sanity test_129 Only applicable to ldiskfs-based MDTs Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 130a: FIEMAP $1-stripe file$ == 00:08:11 $1414393691$ Lustre: DEBUG MARKER: == sanity test 130a: FIEMAP (1-stripe file) == 00:08:11 (1414393691) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_130a ORI-366\/LU-1941: FIEMAP unimplemented on ZFS Lustre: DEBUG MARKER: SKIP: sanity test_130a ORI-366/LU-1941: FIEMAP unimplemented on ZFS Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 130b: FIEMAP $2-stripe file$ == 00:08:14 $1414393694$ Lustre: DEBUG MARKER: == sanity test 130b: FIEMAP (2-stripe file) == 00:08:14 (1414393694) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_130b ORI-366\/LU-1941: FIEMAP unimplemented on ZFS Lustre: DEBUG MARKER: SKIP: sanity test_130b ORI-366/LU-1941: FIEMAP unimplemented on ZFS Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 130c: FIEMAP $2-stripe file with hole$ == 00:08:16 $1414393696$ Lustre: DEBUG MARKER: == sanity test 130c: FIEMAP (2-stripe file with hole) == 00:08:16 (1414393696) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_130c ORI-366\/LU-1941: FIEMAP unimplemented on ZFS Lustre: DEBUG MARKER: SKIP: sanity test_130c ORI-366/LU-1941: FIEMAP unimplemented on ZFS Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 130d: FIEMAP $N-stripe file$ == 00:08:18 $1414393698$ Lustre: DEBUG MARKER: == sanity test 130d: FIEMAP (N-stripe file) == 00:08:18 (1414393698) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_130d skipping FIEMAP on N-stripe file test Lustre: DEBUG MARKER: SKIP: sanity test_130d skipping FIEMAP on N-stripe file test Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 130e: FIEMAP $test continuation FIEMAP calls$ == 00:08:19 $1414393699$ Lustre: DEBUG MARKER: == sanity test 130e: FIEMAP (test continuation FIEMAP calls) == 00:08:19 (1414393699) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_130e ORI-366\/LU-1941: FIEMAP unimplemented on ZFS Lustre: DEBUG MARKER: SKIP: sanity test_130e ORI-366/LU-1941: FIEMAP unimplemented on ZFS Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 131a: test iov\'s crossing stripe boundary for writev\/readv == 00:08:21 $1414393701$ Lustre: DEBUG MARKER: == sanity test 131a: test iov's crossing stripe boundary for writev/readv == 00:08:21 (1414393701) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 131b: test append writev == 00:08:23 $1414393703$ Lustre: DEBUG MARKER: == sanity test 131b: test append writev == 00:08:23 (1414393703) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 131c: test read\/write on file w\/o objects == 00:08:24 $1414393704$ Lustre: DEBUG MARKER: == sanity test 131c: test read/write on file w/o objects == 00:08:24 (1414393704) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 131d: test short read == 00:08:26 $1414393706$ Lustre: DEBUG MARKER: == sanity test 131d: test short read == 00:08:26 (1414393706) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 131e: test read hitting hole == 00:08:27 $1414393707$ Lustre: DEBUG MARKER: == sanity test 131e: test read hitting hole == 00:08:27 (1414393707) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 132: som avoids glimpse rpc == 00:08:29 $1414393709$ Lustre: DEBUG MARKER: == sanity test 132: som avoids glimpse rpc == 00:08:29 (1414393709) Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n ost.OSS.ost.stats Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n ost.OSS.ost.stats LustreError: 11-0: lustre-MDT0000-lwp-OST0001: Communicating with 10.100.4.147@tcp, operation obd_ping failed with -107. Lustre: lustre-MDT0000-lwp-OST0001: Connection to lustre-MDT0000 (at 10.100.4.147@tcp) was lost; in progress operations using this service will wait for recovery to complete Lustre: Skipped 1 previous similar message Lustre: DEBUG MARKER: grep -c /mnt/ost1' ' /proc/mounts Lustre: DEBUG MARKER: umount -d -f /mnt/ost1 Lustre: 8985:0:(client.c:1918:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1414393726/real 1414393726] req@ffff88004c524800 x1483091919084316/t0(0) o400->MGC10.100.4.147@tcp@10.100.4.147@tcp:26/25 lens 224/224 e 0 to 1 dl 1414393733 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 Lustre: 8985:0:(client.c:1918:ptlrpc_expire_one_request()) Skipped 1 previous similar message LustreError: 166-1: MGC10.100.4.147@tcp: Connection to MGS (at 10.100.4.147@tcp) was lost; in progress operations using this service will fail Lustre: server umount lustre-OST0000 complete Lustre: DEBUG MARKER: lsmod | grep lnet > /dev/null && lctl dl | grep ' ST ' Lustre: DEBUG MARKER: ! zpool list -H lustre-ost1 >/dev/null 2>&1 || grep -q ^lustre-ost1/ /proc/mounts || zpool export lustre-ost1 Lustre: DEBUG MARKER: grep -c /mnt/ost2' ' /proc/mounts Lustre: DEBUG MARKER: umount -d -f /mnt/ost2 Lustre: 8984:0:(client.c:1918:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1414393733/real 1414393733] req@ffff880029319000 x1483091919084332/t0(0) o250->MGC10.100.4.147@tcp@10.100.4.147@tcp:26/25 lens 400/544 e 0 to 1 dl 1414393739 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 Lustre: server umount lustre-OST0001 complete Lustre: DEBUG MARKER: lsmod | grep lnet > /dev/null && lctl dl | grep ' ST ' Lustre: DEBUG MARKER: ! zpool list -H lustre-ost2 >/dev/null 2>&1 || grep -q ^lustre-ost2/ /proc/mounts || zpool export lustre-ost2 Lustre: DEBUG MARKER: running=$(grep -c /mnt/ost1' ' /proc/mounts); mpts=$(mount | grep -c /mnt/ost1' '); if [ $running -ne $mpts ]; then echo $(hostname) env are INSANE!; exit 1; fi Lustre: DEBUG MARKER: running=$(grep -c /mnt/ost2' ' /proc/mounts); mpts=$(mount | grep -c /mnt/ost2' '); if [ $running -ne $mpts ]; then echo $(hostname) env are INSANE!; exit 1; fi Lustre: DEBUG MARKER: mkdir -p /mnt/ost1 Lustre: DEBUG MARKER: zpool list -H lustre-ost1 >/dev/null 2>&1 || zpool import -f -o cachefile=none -d /dev lustre-ost1 Lustre: DEBUG MARKER: mkdir -p /mnt/ost1; mount -t lustre lustre-ost1/ost1 /mnt/ost1 LustreError: 137-5: lustre-OST0001_UUID: not available for connect from 10.100.4.147@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. Lustre: DEBUG MARKER: PATH=/usr/lib64/lustre/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:/opt/iozone/bin:/opt/iozone/bin:/usr/lib64/lustre/tests/mpi:/usr/lib64/lustre/tests/racer:/usr/lib64/lustre/../lustre-iokit/sgpdd-survey:/usr/lib64/lustre/tests:/usr/lib64/lustre/u Lustre: DEBUG MARKER: zfs get -H -o value lustre:svname lustre-ost1/ost1 2>/dev/null Lustre: DEBUG MARKER: mkdir -p /mnt/ost2 Lustre: DEBUG MARKER: zpool list -H lustre-ost2 >/dev/null 2>&1 || zpool import -f -o cachefile=none -d /dev lustre-ost2 Lustre: lustre-OST0000: deleting orphan objects from 0x0:25917 to 0x0:25953 Lustre: DEBUG MARKER: mkdir -p /mnt/ost2; mount -t lustre lustre-ost2/ost2 /mnt/ost2 LustreError: 137-5: lustre-OST0001_UUID: not available for connect from 10.100.4.147@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. Lustre: DEBUG MARKER: PATH=/usr/lib64/lustre/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:/opt/iozone/bin:/opt/iozone/bin:/usr/lib64/lustre/tests/mpi:/usr/lib64/lustre/tests/racer:/usr/lib64/lustre/../lustre-iokit/sgpdd-survey:/usr/lib64/lustre/tests:/usr/lib64/lustre/u Lustre: DEBUG MARKER: zfs get -H -o value lustre:svname lustre-ost2/ost2 2>/dev/null Lustre: lustre-OST0001: deleting orphan objects from 0x0:20881 to 0x0:20897 Lustre: DEBUG MARKER: /usr/sbin/lctl mark Using TIMEOUT=20 Lustre: DEBUG MARKER: Using TIMEOUT=20 Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-zfs.lustre-OST0000.quota_slave.enabled Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n ost.OSS.ost.stats Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n ost.OSS.ost.stats LustreError: 11-0: lustre-MDT0000-lwp-OST0001: Communicating with 10.100.4.147@tcp, operation obd_ping failed with -107. LustreError: Skipped 1 previous similar message Lustre: lustre-MDT0000-lwp-OST0001: Connection to lustre-MDT0000 (at 10.100.4.147@tcp) was lost; in progress operations using this service will wait for recovery to complete Lustre: Skipped 1 previous similar message Lustre: DEBUG MARKER: grep -c /mnt/ost1' ' /proc/mounts Lustre: DEBUG MARKER: umount -d -f /mnt/ost1 Lustre: server umount lustre-OST0000 complete Lustre: DEBUG MARKER: lsmod | grep lnet > /dev/null && lctl dl | grep ' ST ' Lustre: DEBUG MARKER: ! zpool list -H lustre-ost1 >/dev/null 2>&1 || grep -q ^lustre-ost1/ /proc/mounts || zpool export lustre-ost1 Lustre: DEBUG MARKER: grep -c /mnt/ost2' ' /proc/mounts Lustre: DEBUG MARKER: umount -d -f /mnt/ost2 Lustre: 8984:0:(client.c:1918:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1414393800/real 1414393800] req@ffff880025f6a000 x1483091919084696/t0(0) o38->lustre-MDT0000-lwp-OST0001@10.100.4.147@tcp:12/10 lens 400/544 e 0 to 1 dl 1414393811 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 Lustre: 8984:0:(client.c:1918:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1414393815/real 1414393815] req@ffff880025b4b000 x1483091919084712/t0(0) o38->lustre-MDT0000-lwp-OST0001@10.100.4.147@tcp:12/10 lens 400/544 e 0 to 1 dl 1414393831 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 Lustre: 10784:0:(client.c:1918:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1414393794/real 1414393794] req@ffff880025b4b800 x1483091919084688/t0(0) o101->MGC10.100.4.147@tcp@10.100.4.147@tcp:26/25 lens 328/344 e 0 to 1 dl 1414393839 ref 2 fl Rpc:X/0/ffffffff rc 0/-1 LustreError: 166-1: MGC10.100.4.147@tcp: Connection to MGS (at 10.100.4.147@tcp) was lost; in progress operations using this service will fail Lustre: server umount lustre-OST0001 complete Lustre: DEBUG MARKER: lsmod | grep lnet > /dev/null && lctl dl | grep ' ST ' Lustre: DEBUG MARKER: ! zpool list -H lustre-ost2 >/dev/null 2>&1 || grep -q ^lustre-ost2/ /proc/mounts || zpool export lustre-ost2 Lustre: DEBUG MARKER: running=$(grep -c /mnt/ost1' ' /proc/mounts); mpts=$(mount | grep -c /mnt/ost1' '); if [ $running -ne $mpts ]; then echo $(hostname) env are INSANE!; exit 1; fi Lustre: DEBUG MARKER: running=$(grep -c /mnt/ost2' ' /proc/mounts); mpts=$(mount | grep -c /mnt/ost2' '); if [ $running -ne $mpts ]; then echo $(hostname) env are INSANE!; exit 1; fi Lustre: DEBUG MARKER: mkdir -p /mnt/ost1 Lustre: DEBUG MARKER: zpool list -H lustre-ost1 >/dev/null 2>&1 || zpool import -f -o cachefile=none -d /dev lustre-ost1 Lustre: DEBUG MARKER: mkdir -p /mnt/ost1; mount -t lustre lustre-ost1/ost1 /mnt/ost1 Lustre: DEBUG MARKER: PATH=/usr/lib64/lustre/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:/opt/iozone/bin:/opt/iozone/bin:/usr/lib64/lustre/tests/mpi:/usr/lib64/lustre/tests/racer:/usr/lib64/lustre/../lustre-iokit/sgpdd-survey:/usr/lib64/lustre/tests:/usr/lib64/lustre/u Lustre: DEBUG MARKER: zfs get -H -o value lustre:svname lustre-ost1/ost1 2>/dev/null Lustre: DEBUG MARKER: mkdir -p /mnt/ost2 LustreError: 137-5: lustre-OST0001_UUID: not available for connect from 10.100.4.147@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. Lustre: DEBUG MARKER: zpool list -H lustre-ost2 >/dev/null 2>&1 || zpool import -f -o cachefile=none -d /dev lustre-ost2 Lustre: DEBUG MARKER: mkdir -p /mnt/ost2; mount -t lustre lustre-ost2/ost2 /mnt/ost2 Lustre: lustre-OST0000: deleting orphan objects from 0x0:25955 to 0x0:25985 Lustre: DEBUG MARKER: PATH=/usr/lib64/lustre/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:/opt/iozone/bin:/opt/iozone/bin:/usr/lib64/lustre/tests/mpi:/usr/lib64/lustre/tests/racer:/usr/lib64/lustre/../lustre-iokit/sgpdd-survey:/usr/lib64/lustre/tests:/usr/lib64/lustre/u Lustre: DEBUG MARKER: zfs get -H -o value lustre:svname lustre-ost2/ost2 2>/dev/null Lustre: DEBUG MARKER: /usr/sbin/lctl mark Using TIMEOUT=20 Lustre: DEBUG MARKER: Using TIMEOUT=20 Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-zfs.lustre-OST0000.quota_slave.enabled Lustre: lustre-OST0001: deleting orphan objects from 0x0:20881 to 0x0:20929 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 133a: Verifying MDT stats ========================================== 00:11:27 $1414393887$ Lustre: DEBUG MARKER: == sanity test 133a: Verifying MDT stats ========================================== 00:11:27 (1414393887) Lustre: DEBUG MARKER: /usr/sbin/lctl set_param obdfilter.*.stats=clear Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 133b: Verifying extra MDT stats ==================================== 00:11:32 $1414393892$ Lustre: DEBUG MARKER: == sanity test 133b: Verifying extra MDT stats ==================================== 00:11:32 (1414393892) Lustre: DEBUG MARKER: /usr/sbin/lctl set_param obdfilter.*.stats=clear Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 133c: Verifying OST stats ========================================== 00:11:35 $1414393895$ Lustre: DEBUG MARKER: == sanity test 133c: Verifying OST stats ========================================== 00:11:35 (1414393895) Lustre: DEBUG MARKER: /usr/sbin/lctl set_param obdfilter.*.stats=clear Lustre: DEBUG MARKER: /usr/sbin/lctl get_param obdfilter.lustre-OST0000.stats Lustre: DEBUG MARKER: /usr/sbin/lctl get_param obdfilter.lustre-OST0000.stats Lustre: DEBUG MARKER: /usr/sbin/lctl get_param obdfilter.lustre-OST0000.stats Lustre: DEBUG MARKER: /usr/sbin/lctl get_param obdfilter.lustre-OST0000.stats Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 133d: Verifying rename_stats ========================================== 00:12:02 $1414393922$ Lustre: DEBUG MARKER: == sanity test 133d: Verifying rename_stats ========================================== 00:12:02 (1414393922) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 133e: Verifying OST {read,write}_bytes nid stats =================== 00:12:10 $1414393930$ Lustre: DEBUG MARKER: == sanity test 133e: Verifying OST read_bytes write_bytes nid stats =================== 00:12:10 (1414393930) Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.*.exports.clear=clear Lustre: DEBUG MARKER: /usr/sbin/lctl get_param obdfilter.*.exports.*.stats Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.*.exports.clear=clear Lustre: DEBUG MARKER: /usr/sbin/lctl get_param obdfilter.*.exports.*.stats Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 133f: Check for LBUGs\/Oopses\/unreadable files in \/proc == 00:12:14 $1414393934$ Lustre: DEBUG MARKER: == sanity test 133f: Check for LBUGs/Oopses/unreadable files in /proc == 00:12:14 (1414393934) Lustre: DEBUG MARKER: find /proc/fs/lustre/ /proc/sys/lnet/ /proc/sys/lustre/ ! -name req_history -exec cat {} \; Lustre: DEBUG MARKER: find /proc/fs/lustre/ /proc/sys/lnet/ /proc/sys/lustre/ ! -name req_history -type f -exec cat {} \; Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 140: Check reasonable stack depth $shouldn\'t LBUG$ ====== 00:12:19 $1414393939$ Lustre: DEBUG MARKER: == sanity test 140: Check reasonable stack depth (shouldn't LBUG) ====== 00:12:19 (1414393939) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 150: truncate\/append tests == 00:12:21 $1414393941$ Lustre: DEBUG MARKER: == sanity test 150: truncate/append tests == 00:12:21 (1414393941) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 151: test cache on oss and controls ================================= 00:12:27 $1414393947$ Lustre: DEBUG MARKER: == sanity test 151: test cache on oss and controls ================================= 00:12:27 (1414393947) Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n obdfilter.lustre-OST*.read_cache_enable osd-*.lustre-OST*.read_cache_enable 2>&1 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 152: test read\/write with enomem ============================== 00:12:29 $1414393949$ Lustre: DEBUG MARKER: == sanity test 152: test read/write with enomem ============================== 00:12:29 (1414393949) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 153: test if fdatasync does not crash ========================= 00:12:31 $1414393951$ Lustre: DEBUG MARKER: == sanity test 153: test if fdatasync does not crash ========================= 00:12:31 (1414393951) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 154A: lfs path2fid and fid2path basic checks == 00:12:35 $1414393955$ Lustre: DEBUG MARKER: == sanity test 154A: lfs path2fid and fid2path basic checks == 00:12:35 (1414393955) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 154a: Open-by-FID == 00:12:36 $1414393956$ Lustre: DEBUG MARKER: == sanity test 154a: Open-by-FID == 00:12:36 (1414393956) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 154b: Open-by-FID for remote directory == 00:12:39 $1414393959$ Lustre: DEBUG MARKER: == sanity test 154b: Open-by-FID for remote directory == 00:12:39 (1414393959) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_154b needs \>= 2 MDTs Lustre: DEBUG MARKER: SKIP: sanity test_154b needs >= 2 MDTs Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 154c: lfs path2fid and fid2path multiple arguments == 00:12:41 $1414393961$ Lustre: DEBUG MARKER: == sanity test 154c: lfs path2fid and fid2path multiple arguments == 00:12:41 (1414393961) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 154d: Verify open file fid == 00:12:43 $1414393963$ Lustre: DEBUG MARKER: == sanity test 154d: Verify open file fid == 00:12:43 (1414393963) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_154d Need MDS version at least 2.5.53 Lustre: DEBUG MARKER: SKIP: sanity test_154d Need MDS version at least 2.5.53 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 155a: Verify small file correctness: read cache:on write_cache:on == 00:12:45 $1414393965$ Lustre: DEBUG MARKER: == sanity test 155a: Verify small file correctness: read cache:on write_cache:on == 00:12:45 (1414393965) Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.read_cache_enable=1 osd-*.lustre-OST*.read_cache_enable=1 2>&1 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.writethrough_cache_enable=1 osd-*.lustre-OST*.writethrough_cache_enable=1 2>&1 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 155b: Verify small file correctness: read cache:on write_cache:off == 00:12:48 $1414393968$ Lustre: DEBUG MARKER: == sanity test 155b: Verify small file correctness: read cache:on write_cache:off == 00:12:48 (1414393968) Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.read_cache_enable=1 osd-*.lustre-OST*.read_cache_enable=1 2>&1 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.writethrough_cache_enable=0 osd-*.lustre-OST*.writethrough_cache_enable=0 2>&1 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 155c: Verify small file correctness: read cache:off write_cache:on == 00:12:51 $1414393971$ Lustre: DEBUG MARKER: == sanity test 155c: Verify small file correctness: read cache:off write_cache:on == 00:12:51 (1414393971) Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.read_cache_enable=0 osd-*.lustre-OST*.read_cache_enable=0 2>&1 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.writethrough_cache_enable=1 osd-*.lustre-OST*.writethrough_cache_enable=1 2>&1 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 155d: Verify small file correctness: read cache:off write_cache:off == 00:12:56 $1414393976$ Lustre: DEBUG MARKER: == sanity test 155d: Verify small file correctness: read cache:off write_cache:off == 00:12:56 (1414393976) Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.read_cache_enable=0 osd-*.lustre-OST*.read_cache_enable=0 2>&1 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.writethrough_cache_enable=0 osd-*.lustre-OST*.writethrough_cache_enable=0 2>&1 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 155e: Verify big file correctness: read cache:on write_cache:on == 00:13:00 $1414393980$ Lustre: DEBUG MARKER: == sanity test 155e: Verify big file correctness: read cache:on write_cache:on == 00:13:00 (1414393980) Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.read_cache_enable=1 osd-*.lustre-OST*.read_cache_enable=1 2>&1 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.writethrough_cache_enable=1 osd-*.lustre-OST*.writethrough_cache_enable=1 2>&1 Lustre: DEBUG MARKER: awk '/cache/ {sum+=$4} END {print sum}' /proc/cpuinfo Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 155f: Verify big file correctness: read cache:on write_cache:off == 00:13:14 $1414393994$ Lustre: DEBUG MARKER: == sanity test 155f: Verify big file correctness: read cache:on write_cache:off == 00:13:14 (1414393994) Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.read_cache_enable=1 osd-*.lustre-OST*.read_cache_enable=1 2>&1 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.writethrough_cache_enable=0 osd-*.lustre-OST*.writethrough_cache_enable=0 2>&1 Lustre: DEBUG MARKER: awk '/cache/ {sum+=$4} END {print sum}' /proc/cpuinfo Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 155g: Verify big file correctness: read cache:off write_cache:on == 00:13:24 $1414394004$ Lustre: DEBUG MARKER: == sanity test 155g: Verify big file correctness: read cache:off write_cache:on == 00:13:24 (1414394004) Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.read_cache_enable=0 osd-*.lustre-OST*.read_cache_enable=0 2>&1 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.writethrough_cache_enable=1 osd-*.lustre-OST*.writethrough_cache_enable=1 2>&1 Lustre: DEBUG MARKER: awk '/cache/ {sum+=$4} END {print sum}' /proc/cpuinfo Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 155h: Verify big file correctness: read cache:off write_cache:off == 00:13:35 $1414394015$ Lustre: DEBUG MARKER: == sanity test 155h: Verify big file correctness: read cache:off write_cache:off == 00:13:35 (1414394015) Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.read_cache_enable=0 osd-*.lustre-OST*.read_cache_enable=0 2>&1 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n obdfilter.lustre-OST*.writethrough_cache_enable=0 osd-*.lustre-OST*.writethrough_cache_enable=0 2>&1 Lustre: DEBUG MARKER: awk '/cache/ {sum+=$4} END {print sum}' /proc/cpuinfo Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 156: Verification of tunables ============================== 00:13:45 $1414394025$ Lustre: DEBUG MARKER: == sanity test 156: Verification of tunables ============================== 00:13:45 (1414394025) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_156 LU-1956\/LU-2261: stats unimplemented on OSD ZFS Lustre: DEBUG MARKER: SKIP: sanity test_156 LU-1956/LU-2261: stats unimplemented on OSD ZFS Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 160a: changelog sanity == 00:13:46 $1414394026$ Lustre: DEBUG MARKER: == sanity test 160a: changelog sanity == 00:13:46 (1414394026) LustreError: 11-0: lustre-MDT0000-lwp-OST0000: Communicating with 10.100.4.147@tcp, operation obd_ping failed with -107. LustreError: Skipped 1 previous similar message Lustre: lustre-MDT0000-lwp-OST0000: Connection to lustre-MDT0000 (at 10.100.4.147@tcp) was lost; in progress operations using this service will wait for recovery to complete Lustre: Skipped 1 previous similar message LustreError: 11-0: MGC10.100.4.147@tcp: Communicating with 10.100.4.147@tcp, operation obd_ping failed with -107. LustreError: Skipped 1 previous similar message LustreError: 166-1: MGC10.100.4.147@tcp: Connection to MGS (at 10.100.4.147@tcp) was lost; in progress operations using this service will fail Lustre: Evicted from MGS (at 10.100.4.147@tcp) after server handle changed from 0xe51f1a4c39027f84 to 0xe51f1a4c3903606b Lustre: MGC10.100.4.147@tcp: Connection restored to MGS (at 10.100.4.147@tcp) Lustre: Skipped 1 previous similar message Lustre: 8984:0:(client.c:1918:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1414394044/real 1414394044] req@ffff880025a9d400 x1483091919085472/t0(0) o38->lustre-MDT0000-lwp-OST0000@10.100.4.147@tcp:12/10 lens 400/544 e 0 to 1 dl 1414394055 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 Lustre: 8984:0:(client.c:1918:ptlrpc_expire_one_request()) Skipped 1 previous similar message Lustre: lustre-OST0000: deleting orphan objects from 0x0:26255 to 0x0:26305 Lustre: lustre-OST0001: deleting orphan objects from 0x0:21202 to 0x0:21217 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 160b: Verify that very long rename doesn\'t crash in changelog == 00:14:16 $1414394056$ Lustre: DEBUG MARKER: == sanity test 160b: Verify that very long rename doesn't crash in changelog == 00:14:16 (1414394056) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 160c: verify that changelog log catch the truncate event == 00:14:18 $1414394058$ Lustre: DEBUG MARKER: == sanity test 160c: verify that changelog log catch the truncate event == 00:14:18 (1414394058) LustreError: 167-0: lustre-MDT0000-lwp-OST0000: This client was evicted by lustre-MDT0000; in progress operations using this service will fail. LustreError: Skipped 1 previous similar message Lustre: lustre-MDT0000-lwp-OST0000: Connection restored to lustre-MDT0000 (at 10.100.4.147@tcp) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 161a: link ea sanity == 00:14:21 $1414394061$ Lustre: DEBUG MARKER: == sanity test 161a: link ea sanity == 00:14:21 (1414394061) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 161b: link ea sanity under remote directory == 00:14:36 $1414394076$ Lustre: DEBUG MARKER: == sanity test 161b: link ea sanity under remote directory == 00:14:36 (1414394076) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_161b skipping remote directory test Lustre: DEBUG MARKER: SKIP: sanity test_161b skipping remote directory test Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 161c: check CL_RENME[UNLINK] changelog record flags == 00:14:37 $1414394077$ Lustre: DEBUG MARKER: == sanity test 161c: check CL_RENME[UNLINK] changelog record flags == 00:14:37 (1414394077) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 162: path lookup sanity == 00:14:40 $1414394080$ Lustre: DEBUG MARKER: == sanity test 162: path lookup sanity == 00:14:40 (1414394080) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 169: parallel read and truncate should not deadlock == 00:14:41 $1414394081$ Lustre: DEBUG MARKER: == sanity test 169: parallel read and truncate should not deadlock == 00:14:41 (1414394081) Lustre: DEBUG MARKER: /usr/sbin/lctl mark creating a 10 Mb file Lustre: DEBUG MARKER: creating a 10 Mb file Lustre: DEBUG MARKER: /usr/sbin/lctl mark starting reads Lustre: DEBUG MARKER: starting reads Lustre: DEBUG MARKER: /usr/sbin/lctl mark truncating the file Lustre: DEBUG MARKER: truncating the file Lustre: DEBUG MARKER: /usr/sbin/lctl mark killing dd Lustre: DEBUG MARKER: killing dd Lustre: DEBUG MARKER: /usr/sbin/lctl mark removing the temporary file Lustre: DEBUG MARKER: removing the temporary file Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 170: test lctl df to handle corrupted log ======================= 00:14:49 $1414394089$ Lustre: DEBUG MARKER: == sanity test 170: test lctl df to handle corrupted log ======================= 00:14:49 (1414394089) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 171: test libcfs_debug_dumplog_thread stuck in do_exit ======== 00:14:53 $1414394093$ Lustre: DEBUG MARKER: == sanity test 171: test libcfs_debug_dumplog_thread stuck in do_exit() ======== 00:14:53 (1414394093) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_180a skipping ALWAYS excluded test 180a $base 180$ Lustre: DEBUG MARKER: SKIP: sanity test_180a skipping ALWAYS excluded test 180a (base 180) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_180b skipping ALWAYS excluded test 180b $base 180$ Lustre: DEBUG MARKER: SKIP: sanity test_180b skipping ALWAYS excluded test 180b (base 180) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_180c skipping ALWAYS excluded test 180c $base 180$ Lustre: DEBUG MARKER: SKIP: sanity test_180c skipping ALWAYS excluded test 180c (base 180) Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 181: Test open-unlinked dir ========================== 00:14:58 $1414394098$ Lustre: DEBUG MARKER: == sanity test 181: Test open-unlinked dir ========================== 00:14:58 (1414394098) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 182: Disable MDC RPCs semaphore wouldn\'t crash client ================== 00:15:37 $1414394137$ Lustre: DEBUG MARKER: == sanity test 182: Disable MDC RPCs semaphore wouldn't crash client ================== 00:15:37 (1414394137) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 183: No crash or request leak in case of strange dispositions ========== 00:15:55 $1414394155$ Lustre: DEBUG MARKER: == sanity test 183: No crash or request leak in case of strange dispositions ========== 00:15:55 (1414394155) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 184a: Basic layout swap == 00:15:58 $1414394158$ Lustre: DEBUG MARKER: == sanity test 184a: Basic layout swap == 00:15:58 (1414394158) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 184b: Forbidden layout swap $will generate errors$ == 00:16:03 $1414394163$ Lustre: DEBUG MARKER: == sanity test 184b: Forbidden layout swap (will generate errors) == 00:16:03 (1414394163) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_184c skipping ALWAYS excluded test 184c Lustre: DEBUG MARKER: SKIP: sanity test_184c skipping ALWAYS excluded test 184c Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 184d: allow stripeless layouts swap == 00:16:06 $1414394166$ Lustre: DEBUG MARKER: == sanity test 184d: allow stripeless layouts swap == 00:16:06 (1414394166) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 185: Volatile file support == 00:16:07 $1414394167$ Lustre: DEBUG MARKER: == sanity test 185: Volatile file support == 00:16:07 (1414394167) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 187a: Test data version change == 00:16:10 $1414394170$ Lustre: DEBUG MARKER: == sanity test 187a: Test data version change == 00:16:10 (1414394170) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 187b: Test data version change on volatile file == 00:16:18 $1414394178$ Lustre: DEBUG MARKER: == sanity test 187b: Test data version change on volatile file == 00:16:18 (1414394178) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 200: OST pools == 00:16:23 $1414394183$ Lustre: DEBUG MARKER: == sanity test 200: OST pools == 00:16:23 (1414394183) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 204a: Print default stripe attributes =================== 00:17:07 $1414394227$ Lustre: DEBUG MARKER: == sanity test 204a: Print default stripe attributes =================== 00:17:07 (1414394227) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 204b: Print default stripe size and offset ============= 00:17:08 $1414394228$ Lustre: DEBUG MARKER: == sanity test 204b: Print default stripe size and offset ============= 00:17:08 (1414394228) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 204c: Print default stripe count and offset ============= 00:17:09 $1414394229$ Lustre: DEBUG MARKER: == sanity test 204c: Print default stripe count and offset ============= 00:17:09 (1414394229) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 204d: Print default stripe count and size =============== 00:17:11 $1414394231$ Lustre: DEBUG MARKER: == sanity test 204d: Print default stripe count and size =============== 00:17:11 (1414394231) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 204e: Print raw stripe attributes =================== 00:17:12 $1414394232$ Lustre: DEBUG MARKER: == sanity test 204e: Print raw stripe attributes =================== 00:17:12 (1414394232) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 204f: Print raw stripe size and offset ============= 00:17:13 $1414394233$ Lustre: DEBUG MARKER: == sanity test 204f: Print raw stripe size and offset ============= 00:17:13 (1414394233) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 204g: Print raw stripe count and offset ============= 00:17:15 $1414394235$ Lustre: DEBUG MARKER: == sanity test 204g: Print raw stripe count and offset ============= 00:17:15 (1414394235) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 204h: Print raw stripe count and size =============== 00:17:16 $1414394236$ Lustre: DEBUG MARKER: == sanity test 204h: Print raw stripe count and size =============== 00:17:16 (1414394236) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 205: Verify job stats == 00:17:17 $1414394237$ Lustre: DEBUG MARKER: == sanity test 205: Verify job stats == 00:17:17 (1414394237) Lustre: DEBUG MARKER: lctl set_param obdfilter.*.job_stats=clear Lustre: DEBUG MARKER: /usr/sbin/lctl mark Test: mkdir \/mnt\/lustre\/f205.sanity Lustre: DEBUG MARKER: Test: mkdir /mnt/lustre/f205.sanity Lustre: DEBUG MARKER: /usr/sbin/lctl mark Using JobID environment variable FAKE_JOBID=test_id.205.24405 Lustre: DEBUG MARKER: Using JobID environment variable FAKE_JOBID=test_id.205.24405 Lustre: DEBUG MARKER: lctl set_param obdfilter.*.job_stats=clear Lustre: DEBUG MARKER: /usr/sbin/lctl mark Test: rm -fr \/mnt\/lustre\/f205.sanity Lustre: DEBUG MARKER: Test: rm -fr /mnt/lustre/f205.sanity Lustre: DEBUG MARKER: /usr/sbin/lctl mark Using JobID environment variable FAKE_JOBID=test_id.205.27502 Lustre: DEBUG MARKER: Using JobID environment variable FAKE_JOBID=test_id.205.27502 Lustre: DEBUG MARKER: lctl set_param obdfilter.*.job_stats=clear Lustre: DEBUG MARKER: /usr/sbin/lctl mark Test: mknod \/mnt\/lustre\/f205.sanity c 1 3 Lustre: DEBUG MARKER: Test: mknod /mnt/lustre/f205.sanity c 1 3 Lustre: DEBUG MARKER: /usr/sbin/lctl mark Using JobID environment variable FAKE_JOBID=test_id.205.3214 Lustre: DEBUG MARKER: Using JobID environment variable FAKE_JOBID=test_id.205.3214 Lustre: DEBUG MARKER: lctl set_param obdfilter.*.job_stats=clear Lustre: DEBUG MARKER: /usr/sbin/lctl mark Test: rm -f \/mnt\/lustre\/f205.sanity Lustre: DEBUG MARKER: Test: rm -f /mnt/lustre/f205.sanity Lustre: DEBUG MARKER: /usr/sbin/lctl mark Using JobID environment variable FAKE_JOBID=test_id.205.21747 Lustre: DEBUG MARKER: Using JobID environment variable FAKE_JOBID=test_id.205.21747 Lustre: DEBUG MARKER: lctl set_param obdfilter.*.job_stats=clear Lustre: DEBUG MARKER: /usr/sbin/lctl mark Test: \/usr\/bin\/lfs setstripe -i 0 -c 1 \/mnt\/lustre\/f205.sanity Lustre: DEBUG MARKER: Test: /usr/bin/lfs setstripe -i 0 -c 1 /mnt/lustre/f205.sanity Lustre: DEBUG MARKER: /usr/sbin/lctl mark Using JobID environment variable FAKE_JOBID=test_id.205.12898 Lustre: DEBUG MARKER: Using JobID environment variable FAKE_JOBID=test_id.205.12898 Lustre: DEBUG MARKER: lctl set_param obdfilter.*.job_stats=clear Lustre: DEBUG MARKER: /usr/sbin/lctl mark Test: touch \/mnt\/lustre\/f205.sanity Lustre: DEBUG MARKER: Test: touch /mnt/lustre/f205.sanity Lustre: DEBUG MARKER: /usr/sbin/lctl mark Using JobID environment variable FAKE_JOBID=test_id.205.17894 Lustre: DEBUG MARKER: Using JobID environment variable FAKE_JOBID=test_id.205.17894 Lustre: DEBUG MARKER: lctl get_param obdfilter.*.job_stats Lustre: DEBUG MARKER: lctl set_param obdfilter.*.job_stats=clear Lustre: DEBUG MARKER: /usr/sbin/lctl mark Test: dd if=\/dev\/zero of=\/mnt\/lustre\/f205.sanity bs=1M count=1 oflag=sync Lustre: DEBUG MARKER: Test: dd if=/dev/zero of=/mnt/lustre/f205.sanity bs=1M count=1 oflag=sync Lustre: DEBUG MARKER: /usr/sbin/lctl mark Using JobID environment variable FAKE_JOBID=test_id.205.9326 Lustre: DEBUG MARKER: Using JobID environment variable FAKE_JOBID=test_id.205.9326 Lustre: DEBUG MARKER: lctl get_param obdfilter.*.job_stats Lustre: DEBUG MARKER: lctl set_param obdfilter.*.job_stats=clear Lustre: DEBUG MARKER: /usr/sbin/lctl mark Test: dd if=\/mnt\/lustre\/f205.sanity of=\/dev\/null bs=1M count=1 iflag=direct Lustre: DEBUG MARKER: Test: dd if=/mnt/lustre/f205.sanity of=/dev/null bs=1M count=1 iflag=direct Lustre: DEBUG MARKER: /usr/sbin/lctl mark Using JobID environment variable FAKE_JOBID=test_id.205.26451 Lustre: DEBUG MARKER: Using JobID environment variable FAKE_JOBID=test_id.205.26451 Lustre: DEBUG MARKER: lctl get_param obdfilter.*.job_stats Lustre: DEBUG MARKER: lctl set_param obdfilter.*.job_stats=clear Lustre: DEBUG MARKER: /usr/sbin/lctl mark Test: \/usr\/lib64\/lustre\/tests\/truncate \/mnt\/lustre\/f205.sanity 0 Lustre: DEBUG MARKER: Test: /usr/lib64/lustre/tests/truncate /mnt/lustre/f205.sanity 0 Lustre: DEBUG MARKER: /usr/sbin/lctl mark Using JobID environment variable FAKE_JOBID=test_id.205.14898 Lustre: DEBUG MARKER: Using JobID environment variable FAKE_JOBID=test_id.205.14898 Lustre: DEBUG MARKER: lctl get_param obdfilter.*.job_stats Lustre: DEBUG MARKER: lctl set_param obdfilter.*.job_stats=clear Lustre: DEBUG MARKER: /usr/sbin/lctl mark Test: mv -f \/mnt\/lustre\/f205.sanity \/mnt\/lustre\/jobstats_test_rename Lustre: DEBUG MARKER: Test: mv -f /mnt/lustre/f205.sanity /mnt/lustre/jobstats_test_rename Lustre: DEBUG MARKER: /usr/sbin/lctl mark Using JobID environment variable FAKE_JOBID=test_id.205.25619 Lustre: DEBUG MARKER: Using JobID environment variable FAKE_JOBID=test_id.205.25619 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 206: fail lov_init_raid0 doesn\'t lbug == 00:17:50 $1414394270$ Lustre: DEBUG MARKER: == sanity test 206: fail lov_init_raid0() doesn't lbug == 00:17:50 (1414394270) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 207a: can refresh layout at glimpse == 00:17:51 $1414394271$ Lustre: DEBUG MARKER: == sanity test 207a: can refresh layout at glimpse == 00:17:51 (1414394271) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 207b: can refresh layout at open == 00:17:53 $1414394273$ Lustre: DEBUG MARKER: == sanity test 207b: can refresh layout at open == 00:17:53 (1414394273) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 208: Exclusive open == 00:17:56 $1414394276$ Lustre: DEBUG MARKER: == sanity test 208: Exclusive open == 00:17:56 (1414394276) LustreError: 11-0: lustre-MDT0000-lwp-OST0001: Communicating with 10.100.4.147@tcp, operation obd_ping failed with -107. Lustre: lustre-MDT0000-lwp-OST0001: Connection to lustre-MDT0000 (at 10.100.4.147@tcp) was lost; in progress operations using this service will wait for recovery to complete Lustre: Skipped 1 previous similar message LustreError: 11-0: MGC10.100.4.147@tcp: Communicating with 10.100.4.147@tcp, operation obd_ping failed with -107. LustreError: Skipped 1 previous similar message LustreError: 166-1: MGC10.100.4.147@tcp: Connection to MGS (at 10.100.4.147@tcp) was lost; in progress operations using this service will fail Lustre: Evicted from MGS (at 10.100.4.147@tcp) after server handle changed from 0xe51f1a4c3903606b to 0xe51f1a4c390f62de Lustre: MGC10.100.4.147@tcp: Connection restored to MGS (at 10.100.4.147@tcp) Lustre: Skipped 1 previous similar message Lustre: lustre-OST0000: deleting orphan objects from 0x0:30337 to 0x0:30369 Lustre: lustre-OST0001: deleting orphan objects from 0x0:25236 to 0x0:25281 Lustre: 8984:0:(client.c:1918:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1414394294/real 1414394294] req@ffff88006dae1400 x1483091919086748/t0(0) o38->lustre-MDT0000-lwp-OST0000@10.100.4.147@tcp:12/10 lens 400/544 e 0 to 1 dl 1414394315 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 Lustre: 8984:0:(client.c:1918:ptlrpc_expire_one_request()) Skipped 1 previous similar message LustreError: 11-0: MGC10.100.4.147@tcp: Communicating with 10.100.4.147@tcp, operation obd_ping failed with -107. LustreError: 166-1: MGC10.100.4.147@tcp: Connection to MGS (at 10.100.4.147@tcp) was lost; in progress operations using this service will fail Lustre: Evicted from MGS (at 10.100.4.147@tcp) after server handle changed from 0xe51f1a4c390f62de to 0xe51f1a4c390f667a Lustre: MGC10.100.4.147@tcp: Connection restored to MGS (at 10.100.4.147@tcp) Lustre: 8984:0:(client.c:1918:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1414394324/real 1414394324] req@ffff880023779c00 x1483091919086888/t0(0) o38->lustre-MDT0000-lwp-OST0000@10.100.4.147@tcp:12/10 lens 400/544 e 0 to 1 dl 1414394349 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 Lustre: 8984:0:(client.c:1918:ptlrpc_expire_one_request()) Skipped 1 previous similar message Lustre: lustre-OST0000: deleting orphan objects from 0x0:30337 to 0x0:30401 Lustre: lustre-OST0001: deleting orphan objects from 0x0:25236 to 0x0:25313 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 209: read-only open\/close requests should be freed promptly == 00:19:12 $1414394352$ Lustre: DEBUG MARKER: == sanity test 209: read-only open/close requests should be freed promptly == 00:19:12 (1414394352) LustreError: 167-0: lustre-MDT0000-lwp-OST0000: This client was evicted by lustre-MDT0000; in progress operations using this service will fail. LustreError: Skipped 1 previous similar message Lustre: lustre-MDT0000-lwp-OST0000: Connection restored to lustre-MDT0000 (at 10.100.4.147@tcp) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 212: Sendfile test ============================================== 00:19:23 $1414394363$ Lustre: DEBUG MARKER: == sanity test 212: Sendfile test ============================================== 00:19:23 (1414394363) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 213: OSC lock completion and cancel race don\'t crash - bug 18829 == 00:19:26 $1414394366$ Lustre: DEBUG MARKER: == sanity test 213: OSC lock completion and cancel race don't crash - bug 18829 == 00:19:26 (1414394366) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 214: hash-indexed directory test - bug 20133 == 00:19:30 $1414394370$ Lustre: DEBUG MARKER: == sanity test 214: hash-indexed directory test - bug 20133 == 00:19:30 (1414394370) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 215: \/proc\/sys\/lnet exists and has proper content - bugs 18102, 21079, 21517 == 00:19:37 $1414394377$ Lustre: DEBUG MARKER: == sanity test 215: /proc/sys/lnet exists and has proper content - bugs 18102, 21079, 21517 == 00:19:37 (1414394377) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 216: check lockless direct write works and updates file size and kms correctly == 00:19:39 $1414394379$ Lustre: DEBUG MARKER: == sanity test 216: check lockless direct write works and updates file size and kms correctly == 00:19:39 (1414394379) Lustre: DEBUG MARKER: /usr/sbin/lctl get_param ldlm.namespaces.filter-*.max_nolock_bytes | while read s; do echo ost1 $s; done Lustre: DEBUG MARKER: /usr/sbin/lctl get_param ldlm.namespaces.filter-*.max_nolock_bytes | while read s; do echo ost2 $s; done Lustre: DEBUG MARKER: /usr/sbin/lctl get_param ldlm.namespaces.filter-*.contended_locks | while read s; do echo ost1 $s; done Lustre: DEBUG MARKER: /usr/sbin/lctl get_param ldlm.namespaces.filter-*.contended_locks | while read s; do echo ost2 $s; done Lustre: DEBUG MARKER: /usr/sbin/lctl get_param ldlm.namespaces.filter-*.contention_seconds | while read s; do echo ost1 $s; done Lustre: DEBUG MARKER: /usr/sbin/lctl get_param ldlm.namespaces.filter-*.contention_seconds | while read s; do echo ost2 $s; done Lustre: DEBUG MARKER: lctl set_param -n ldlm.namespaces.filter-*.max_nolock_bytes 2000000; lctl set_param -n ldlm.namespaces.filter-*.contended_locks 0; lctl set_param -n ldlm.namespaces.filter-*.contention_seconds 60 Lustre: DEBUG MARKER: lctl set_param -n ldlm.namespaces.filter-*.max_nolock_bytes 0; lctl set_param -n ldlm.namespaces.filter-*.contended_locks 32; lctl set_param -n ldlm.namespaces.filter-*.contention_seconds 0 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n ldlm.namespaces.filter-lustre-OST0000_UUID.max_nolock_bytes 0 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n ldlm.namespaces.filter-lustre-OST0001_UUID.max_nolock_bytes 0 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n ldlm.namespaces.filter-lustre-OST0000_UUID.max_nolock_bytes 0 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n ldlm.namespaces.filter-lustre-OST0001_UUID.max_nolock_bytes 0 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n ldlm.namespaces.filter-lustre-OST0000_UUID.contended_locks 32 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n ldlm.namespaces.filter-lustre-OST0001_UUID.contended_locks 32 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n ldlm.namespaces.filter-lustre-OST0000_UUID.contended_locks 32 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n ldlm.namespaces.filter-lustre-OST0001_UUID.contended_locks 32 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n ldlm.namespaces.filter-lustre-OST0000_UUID.contention_seconds 2 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n ldlm.namespaces.filter-lustre-OST0001_UUID.contention_seconds 2 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n ldlm.namespaces.filter-lustre-OST0000_UUID.contention_seconds 2 Lustre: DEBUG MARKER: /usr/sbin/lctl set_param -n ldlm.namespaces.filter-lustre-OST0001_UUID.contention_seconds 2 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 217: check lctl ping for hostnames with hiphen $\'-\'$ == 00:19:46 $1414394386$ Lustre: DEBUG MARKER: == sanity test 217: check lctl ping for hostnames with hiphen ('-') == 00:19:46 (1414394386) Lustre: DEBUG MARKER: /usr/sbin/lctl list_nids | grep tcp | cut -f 1 -d '@' Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 218: parallel read and truncate should not deadlock ========================= 00:19:48 $1414394388$ Lustre: DEBUG MARKER: == sanity test 218: parallel read and truncate should not deadlock ========================= 00:19:48 (1414394388) Lustre: DEBUG MARKER: /usr/sbin/lctl mark creating a 10 Mb file Lustre: DEBUG MARKER: creating a 10 Mb file Lustre: DEBUG MARKER: /usr/sbin/lctl mark starting reads Lustre: DEBUG MARKER: starting reads Lustre: DEBUG MARKER: /usr/sbin/lctl mark truncating the file Lustre: DEBUG MARKER: truncating the file Lustre: DEBUG MARKER: /usr/sbin/lctl mark killing dd Lustre: DEBUG MARKER: killing dd Lustre: DEBUG MARKER: /usr/sbin/lctl mark removing the temporary file Lustre: DEBUG MARKER: removing the temporary file Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 219: LU-394: Write partial won\'t cause uncontiguous pages vec at LND == 00:19:56 $1414394396$ Lustre: DEBUG MARKER: == sanity test 219: LU-394: Write partial won't cause uncontiguous pages vec at LND == 00:19:56 (1414394396) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 220: preallocated MDS objects still used if ENOSPC from OST == 00:19:57 $1414394397$ Lustre: DEBUG MARKER: == sanity test 220: preallocated MDS objects still used if ENOSPC from OST == 00:19:57 (1414394397) Lustre: DEBUG MARKER: lctl set_param fail_val=-1 Lustre: DEBUG MARKER: lctl set_param fail_loc=0x229 Lustre: *** cfs_fail_loc=229, val=1*** Lustre: Skipped 13 previous similar messages Lustre: DEBUG MARKER: lctl set_param fail_val=0 Lustre: DEBUG MARKER: lctl set_param fail_loc=0 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 221: make sure fault and truncate race to not cause OOM == 00:20:31 $1414394431$ Lustre: DEBUG MARKER: == sanity test 221: make sure fault and truncate race to not cause OOM == 00:20:31 (1414394431) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 222a: AGL for ls should not trigger CLIO lock failure ================== 00:23:55 $1414394635$ Lustre: DEBUG MARKER: == sanity test 222a: AGL for ls should not trigger CLIO lock failure ================== 00:23:55 (1414394635) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 222b: AGL for rmdir should not trigger CLIO lock failure =============== 00:23:57 $1414394637$ Lustre: DEBUG MARKER: == sanity test 222b: AGL for rmdir should not trigger CLIO lock failure =============== 00:23:57 (1414394637) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 223: osc reenqueue if without AGL lock granted ========================= 00:23:58 $1414394638$ Lustre: DEBUG MARKER: == sanity test 223: osc reenqueue if without AGL lock granted ========================= 00:23:58 (1414394638) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 224a: Don\'t panic on bulk IO failure == 00:24:00 $1414394640$ Lustre: DEBUG MARKER: == sanity test 224a: Don't panic on bulk IO failure == 00:24:00 (1414394640) Lustre: lustre-OST0000: Client adf31587-e977-21cb-98a9-70e8fc22104b (at 10.100.4.141@tcp) reconnecting Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 224b: Don\'t panic on bulk IO failure == 00:24:02 $1414394642$ Lustre: DEBUG MARKER: == sanity test 224b: Don't panic on bulk IO failure == 00:24:02 (1414394642) Lustre: lustre-OST0001: Client adf31587-e977-21cb-98a9-70e8fc22104b (at 10.100.4.141@tcp) reconnecting Lustre: lustre-OST0001: Client adf31587-e977-21cb-98a9-70e8fc22104b (at 10.100.4.141@tcp) refused reconnection, still busy with 1 active RPCs LustreError: 14697:0:(ldlm_lib.c:2707:target_bulk_io()) @@@ Reconnect on bulk PUT req@ffff880075ea2000 x1483091859592312/t0(0) o3->adf31587-e977-21cb-98a9-70e8fc22104b@10.100.4.141@tcp:0/0 lens 488/432 e 0 to 0 dl 1414394687 ref 1 fl Interpret:/0/0 rc 0/0 Lustre: lustre-OST0001: Bulk IO read error with adf31587-e977-21cb-98a9-70e8fc22104b (at 10.100.4.141@tcp), client will retry: rc -110 Lustre: lustre-OST0001: Client adf31587-e977-21cb-98a9-70e8fc22104b (at 10.100.4.141@tcp) reconnecting Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 225a: Metadata survey sanity with zero-stripe == 00:24:09 $1414394649$ Lustre: DEBUG MARKER: == sanity test 225a: Metadata survey sanity with zero-stripe == 00:24:09 (1414394649) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 225b: Metadata survey sanity with stripe_count = 1 == 00:24:19 $1414394659$ Lustre: DEBUG MARKER: == sanity test 225b: Metadata survey sanity with stripe_count = 1 == 00:24:19 (1414394659) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 226a: call path2fid and fid2path on files of all type == 00:24:28 $1414394668$ Lustre: DEBUG MARKER: == sanity test 226a: call path2fid and fid2path on files of all type == 00:24:28 (1414394668) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 226b: call path2fid and fid2path on files of all type under remote dir == 00:24:30 $1414394670$ Lustre: DEBUG MARKER: == sanity test 226b: call path2fid and fid2path on files of all type under remote dir == 00:24:30 (1414394670) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_226b needs \>= 2 MDTs Lustre: DEBUG MARKER: SKIP: sanity test_226b needs >= 2 MDTs Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 227: running truncated executable does not cause OOM == 00:24:31 $1414394671$ Lustre: DEBUG MARKER: == sanity test 227: running truncated executable does not cause OOM == 00:24:31 (1414394671) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 228a: try to reuse idle OI blocks == 00:24:33 $1414394673$ Lustre: DEBUG MARKER: == sanity test 228a: try to reuse idle OI blocks == 00:24:33 (1414394673) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_228a non-ldiskfs backend Lustre: DEBUG MARKER: SKIP: sanity test_228a non-ldiskfs backend Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 228b: idle OI blocks can be reused after MDT restart == 00:24:35 $1414394675$ Lustre: DEBUG MARKER: == sanity test 228b: idle OI blocks can be reused after MDT restart == 00:24:35 (1414394675) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_228b non-ldiskfs backend Lustre: DEBUG MARKER: SKIP: sanity test_228b non-ldiskfs backend Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 228c: NOT shrink the last entry in OI index node to recycle idle leaf == 00:24:36 $1414394676$ Lustre: DEBUG MARKER: == sanity test 228c: NOT shrink the last entry in OI index node to recycle idle leaf == 00:24:36 (1414394676) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_228c non-ldiskfs backend Lustre: DEBUG MARKER: SKIP: sanity test_228c non-ldiskfs backend Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 229: getstripe\/stat\/rm\/attr changes work on released files == 00:24:38 $1414394678$ Lustre: DEBUG MARKER: == sanity test 229: getstripe/stat/rm/attr changes work on released files == 00:24:38 (1414394678) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 230a: Create remote directory and files under the remote directory == 00:24:40 $1414394680$ Lustre: DEBUG MARKER: == sanity test 230a: Create remote directory and files under the remote directory == 00:24:40 (1414394680) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_230a needs \>= 2 MDTs Lustre: DEBUG MARKER: SKIP: sanity test_230a needs >= 2 MDTs Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 230b: nested remote directory should be failed == 00:24:42 $1414394682$ Lustre: DEBUG MARKER: == sanity test 230b: nested remote directory should be failed == 00:24:42 (1414394682) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_230b needs \>= 2 MDTs Lustre: DEBUG MARKER: SKIP: sanity test_230b needs >= 2 MDTs Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 231a: checking that reading\/writing of BRW RPC size results in one RPC == 00:24:43 $1414394683$ Lustre: DEBUG MARKER: == sanity test 231a: checking that reading/writing of BRW RPC size results in one RPC == 00:24:43 (1414394683) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 231b: must not assert on fully utilized OST request buffer == 00:24:45 $1414394685$ Lustre: DEBUG MARKER: == sanity test 231b: must not assert on fully utilized OST request buffer == 00:24:45 (1414394685) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 232: failed lock should not block umount == 00:24:57 $1414394697$ Lustre: DEBUG MARKER: == sanity test 232: failed lock should not block umount == 00:24:57 (1414394697) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 233a: checking that OBF of the FS root succeeds == 00:24:58 $1414394698$ Lustre: DEBUG MARKER: == sanity test 233a: checking that OBF of the FS root succeeds == 00:24:58 (1414394698) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 233b: checking that OBF of the FS .lustre succeeds == 00:25:00 $1414394700$ Lustre: DEBUG MARKER: == sanity test 233b: checking that OBF of the FS .lustre succeeds == 00:25:00 (1414394700) Lustre: DEBUG MARKER: /usr/sbin/lctl mark SKIP: sanity test_233b Need MDS version at least 2.5.90 Lustre: DEBUG MARKER: SKIP: sanity test_233b Need MDS version at least 2.5.90 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 234: xattr cache should not crash on ENOMEM == 00:25:02 $1414394702$ Lustre: DEBUG MARKER: == sanity test 234: xattr cache should not crash on ENOMEM == 00:25:02 (1414394702) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 235: LU-1715: flock deadlock detection does not work properly == 00:25:03 $1414394703$ Lustre: DEBUG MARKER: == sanity test 235: LU-1715: flock deadlock detection does not work properly == 00:25:03 (1414394703) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 236: Layout swap on open unlinked file == 00:25:07 $1414394707$ Lustre: DEBUG MARKER: == sanity test 236: Layout swap on open unlinked file == 00:25:07 (1414394707) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 238: Verify linkea consistency == 00:25:12 $1414394712$ Lustre: DEBUG MARKER: == sanity test 238: Verify linkea consistency == 00:25:12 (1414394712) Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test 900: umount should not race with any mgc requeue thread == 00:25:13 $1414394713$ Lustre: DEBUG MARKER: == sanity test 900: umount should not race with any mgc requeue thread == 00:25:13 (1414394713) LustreError: 11-0: lustre-MDT0000-lwp-OST0001: Communicating with 10.100.4.147@tcp, operation obd_ping failed with -107. Lustre: lustre-MDT0000-lwp-OST0001: Connection to lustre-MDT0000 (at 10.100.4.147@tcp) was lost; in progress operations using this service will wait for recovery to complete Lustre: Skipped 1 previous similar message Lustre: DEBUG MARKER: grep -c /mnt/ost1' ' /proc/mounts Lustre: DEBUG MARKER: umount -d -f /mnt/ost1 Lustre: server umount lustre-OST0000 complete Lustre: DEBUG MARKER: lsmod | grep lnet > /dev/null && lctl dl | grep ' ST ' Lustre: DEBUG MARKER: ! zpool list -H lustre-ost1 >/dev/null 2>&1 || grep -q ^lustre-ost1/ /proc/mounts || zpool export lustre-ost1 Lustre: DEBUG MARKER: grep -c /mnt/ost2' ' /proc/mounts Lustre: DEBUG MARKER: umount -d -f /mnt/ost2 Lustre: 1685:0:(client.c:1918:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1414394737/real 1414394737] req@ffff880023e12000 x1483091919088384/t0(0) o251->MGC10.100.4.147@tcp@10.100.4.147@tcp:26/25 lens 224/224 e 0 to 1 dl 1414394743 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 Lustre: 1685:0:(client.c:1918:ptlrpc_expire_one_request()) Skipped 1 previous similar message Lustre: server umount lustre-OST0001 complete Lustre: DEBUG MARKER: lsmod | grep lnet > /dev/null && lctl dl | grep ' ST ' Lustre: DEBUG MARKER: ! zpool list -H lustre-ost2 >/dev/null 2>&1 || grep -q ^lustre-ost2/ /proc/mounts || zpool export lustre-ost2 Lustre: DEBUG MARKER: running=$(grep -c /mnt/ost1' ' /proc/mounts); mpts=$(mount | grep -c /mnt/ost1' '); if [ $running -ne $mpts ]; then echo $(hostname) env are INSANE!; exit 1; fi Lustre: DEBUG MARKER: running=$(grep -c /mnt/ost2' ' /proc/mounts); mpts=$(mount | grep -c /mnt/ost2' '); if [ $running -ne $mpts ]; then echo $(hostname) env are INSANE!; exit 1; fi Lustre: DEBUG MARKER: mkdir -p /mnt/ost1 Lustre: DEBUG MARKER: zpool list -H lustre-ost1 >/dev/null 2>&1 || zpool import -f -o cachefile=none -d /dev lustre-ost1 Lustre: DEBUG MARKER: mkdir -p /mnt/ost1; mount -t lustre lustre-ost1/ost1 /mnt/ost1 Lustre: DEBUG MARKER: PATH=/usr/lib64/lustre/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:/opt/iozone/bin:/opt/iozone/bin:/usr/lib64/lustre/tests/mpi:/usr/lib64/lustre/tests/racer:/usr/lib64/lustre/../lustre-iokit/sgpdd-survey:/usr/lib64/lustre/tests:/usr/lib64/lustre/u Lustre: DEBUG MARKER: zfs get -H -o value lustre:svname lustre-ost1/ost1 2>/dev/null Lustre: DEBUG MARKER: mkdir -p /mnt/ost2 Lustre: DEBUG MARKER: zpool list -H lustre-ost2 >/dev/null 2>&1 || zpool import -f -o cachefile=none -d /dev lustre-ost2 LustreError: 137-5: lustre-OST0001_UUID: not available for connect from 10.100.4.147@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. Lustre: DEBUG MARKER: mkdir -p /mnt/ost2; mount -t lustre lustre-ost2/ost2 /mnt/ost2 Lustre: lustre-OST0000: deleting orphan objects from 0x0:31128 to 0x0:31649 Lustre: DEBUG MARKER: PATH=/usr/lib64/lustre/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:/opt/iozone/bin:/opt/iozone/bin:/usr/lib64/lustre/tests/mpi:/usr/lib64/lustre/tests/racer:/usr/lib64/lustre/../lustre-iokit/sgpdd-survey:/usr/lib64/lustre/tests:/usr/lib64/lustre/u Lustre: DEBUG MARKER: zfs get -H -o value lustre:svname lustre-ost2/ost2 2>/dev/null Lustre: DEBUG MARKER: /usr/sbin/lctl mark Using TIMEOUT=20 Lustre: DEBUG MARKER: Using TIMEOUT=20 Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-zfs.lustre-OST0000.quota_slave.enabled Lustre: lustre-OST0001: deleting orphan objects from 0x0:25996 to 0x0:27521 Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity test complete, duration 8083 sec == 00:26:42 $1414394802$ Lustre: DEBUG MARKER: == sanity test complete, duration 8083 sec == 00:26:42 (1414394802) Lustre: DEBUG MARKER: /usr/sbin/lctl mark -----============= acceptance-small: sanity-benchmark ============----- Mon Oct 27 00:27:09 PDT 2014 Lustre: DEBUG MARKER: -----============= acceptance-small: sanity-benchmark ============----- Mon Oct 27 00:27:09 PDT 2014 Lustre: DEBUG MARKER: /usr/sbin/lctl mark excepting tests: Lustre: DEBUG MARKER: excepting tests: Lustre: DEBUG MARKER: running=$(grep -c /mnt/ost1' ' /proc/mounts); mpts=$(mount | grep -c /mnt/ost1' '); if [ $running -ne $mpts ]; then echo $(hostname) env are INSANE!; exit 1; fi Lustre: DEBUG MARKER: running=$(grep -c /mnt/ost2' ' /proc/mounts); mpts=$(mount | grep -c /mnt/ost2' '); if [ $running -ne $mpts ]; then echo $(hostname) env are INSANE!; exit 1; fi Lustre: DEBUG MARKER: zfs get -H -o value lustre:svname lustre-ost1/ost1 2>/dev/null Lustre: DEBUG MARKER: cat /proc/mounts Lustre: DEBUG MARKER: zfs get -H -o value lustre:svname lustre-ost2/ost2 2>/dev/null Lustre: DEBUG MARKER: cat /proc/mounts Lustre: DEBUG MARKER: /usr/sbin/lctl mark Using TIMEOUT=20 Lustre: DEBUG MARKER: Using TIMEOUT=20 Lustre: DEBUG MARKER: /usr/sbin/lctl get_param -n osd-zfs.lustre-OST0000.quota_slave.enabled Lustre: DEBUG MARKER: PATH=/usr/lib64/lustre/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:/opt/iozone/bin:/opt/iozone/bin:/usr/lib64/lustre/tests/mpi:/usr/lib64/lustre/tests/racer:/usr/lib64/lustre/../lustre-iokit/sgpdd-survey:/usr/lib64/lustre/tests:/usr/lib64/lustre/u Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity-benchmark test dbench: dbench == 00:27:22 $1414394842$ Lustre: DEBUG MARKER: == sanity-benchmark test dbench: dbench == 00:27:22 (1414394842) LNet: Service thread pid 2304 completed after 64.23s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). LNet: Service thread pid 3368 completed after 78.49s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). LNet: Skipped 3 previous similar messages LNet: Service thread pid 3522 completed after 86.15s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). LNet: Skipped 7 previous similar messages Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity-benchmark test bonnie: bonnie++ == 00:51:35 $1414396295$ Lustre: DEBUG MARKER: == sanity-benchmark test bonnie: bonnie++ == 00:51:35 (1414396295) Lustre: DEBUG MARKER: /usr/sbin/lctl mark min OST has 64814208kB available, using 3845048kB file size Lustre: DEBUG MARKER: min OST has 64814208kB available, using 3845048kB file size Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity-benchmark test iozone: iozone == 01:12:05 $1414397525$ Lustre: DEBUG MARKER: == sanity-benchmark test iozone: iozone == 01:12:05 (1414397525) Lustre: DEBUG MARKER: /usr/sbin/lctl mark min OST has 65026304kB available, using 3845048kB file size Lustre: DEBUG MARKER: min OST has 65026304kB available, using 3845048kB file size LNet: Service thread pid 3523 completed after 51.77s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). LNet: Service thread pid 2305 completed after 52.43s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity-benchmark test fsx: fsx == 05:38:24 $1414413504$ Lustre: DEBUG MARKER: == sanity-benchmark test fsx: fsx == 05:38:24 (1414413504) LNet: Service thread pid 3367 completed after 42.45s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity-benchmark test pios_ssf: pios shared single file == 05:55:33 $1414414533$ Lustre: DEBUG MARKER: == sanity-benchmark test pios_ssf: pios shared single file == 05:55:33 (1414414533) Lustre: DEBUG MARKER: /usr/sbin/lctl mark \/usr\/bin\/pios -t 1,8,40 -n 1024 -c 1M -s 8M -o 16M -p \/mnt\/lustre\/dpios_ssf.sanity-benchmark Lustre: DEBUG MARKER: /usr/bin/pios -t 1,8,40 -n 1024 -c 1M -s 8M -o 16M -p /mnt/lustre/dpios_ssf.sanity-benchmark Lustre: DEBUG MARKER: /usr/sbin/lctl mark \/usr\/bin\/pios -t 1,8,40 -n 1024 -c 1M -s 8M -o 16M --verify -p \/mnt\/lustre\/dpios_ssf.sanity-benchmark Lustre: DEBUG MARKER: /usr/bin/pios -t 1,8,40 -n 1024 -c 1M -s 8M -o 16M --verify -p /mnt/lustre/dpios_ssf.sanity-benchmark Lustre: DEBUG MARKER: lctl set_param -n fail_loc=0 2>/dev/null || true Lustre: DEBUG MARKER: rc=$([ -f /proc/sys/lnet/catastrophe ] && echo $(< /proc/sys/lnet/catastrophe) || echo 0); if [ $rc -ne 0 ]; then echo $(hostname): $rc; fi exit $rc Lustre: DEBUG MARKER: /usr/sbin/lctl mark == sanity-benchmark test pios_fpp: pios file per process == 06:54:04 $1414418044$ Lustre: DEBUG MARKER: == sanity-benchmark test pios_fpp: pios file per process == 06:54:04 (1414418044) Lustre: DEBUG MARKER: /usr/sbin/lctl mark \/usr\/bin\/pios -t 1,8,40 -n 1024 -c 1M -s 8M -o 16M -L fpp -p \/mnt\/lustre\/dpios_fpp.sanity-benchmark Lustre: DEBUG MARKER: /usr/bin/pios -t 1,8,40 -n 1024 -c 1M -s 8M -o 16M -L fpp -p /mnt/lustre/dpios_fpp.sanity-benchmark Lustre: lustre-OST0001: Slow creates, 128/256 objects created at a rate of 2/s LNet: Service thread pid 3372 completed after 91.53s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). LNet: Skipped 15 previous similar messages Lustre: DEBUG MARKER: /usr/sbin/lctl mark \/usr\/bin\/pios -t 1,8,40 -n 1024 -c 1M -s 8M -o 16M -L fpp --verify -p \/mnt\/lustre\/dpios_fpp.sanity-benchmark Lustre: DEBUG MARKER: /usr/bin/pios -t 1,8,40 -n 1024 -c 1M -s 8M -o 16M -L fpp --verify -p /mnt/lustre/dpios_fpp.sanity-benchmark spl_kmem_cache/ invoked oom-killer: gfp_mask=0x84d0, order=0, oom_adj=0, oom_score_adj=0 spl_kmem_cache/ cpuset=/ mems_allowed=0 Pid: 396, comm: spl_kmem_cache/ Tainted: P --------------- 2.6.32-431.29.2.el6_lustre.g9835a2a.x86_64 #1 Call Trace: [] ? cpuset_print_task_mems_allowed+0x91/0xb0 [] ? dump_header+0x90/0x1b0 [] ? check_panic_on_oom+0x4e/0x80 [] ? out_of_memory+0x1bb/0x3c0 [] ? __alloc_pages_nodemask+0x89f/0x8d0 [] ? alloc_pages_current+0xaa/0x110 [] ? __get_free_pages+0xe/0x50 [] ? pte_alloc_one_kernel+0x15/0x20 [] ? __pte_alloc_kernel+0x1b/0xc0 [] ? vmap_page_range_noflush+0x309/0x370 [] ? map_vm_area+0x32/0x50 [] ? __vmalloc_area_node+0x100/0x190 [] ? kv_alloc+0x59/0x60 [spl] [] ? __vmalloc_node+0xad/0x120 [] ? kv_alloc+0x59/0x60 [spl] [] ? __vmalloc+0x22/0x30 [] ? kv_alloc+0x59/0x60 [spl] [] ? spl_cache_grow_work+0x39/0x2d0 [spl] [] ? __wake_up+0x53/0x70 [] ? taskq_thread+0x1e7/0x3f0 [spl] [] ? default_wake_function+0x0/0x20 [] ? taskq_thread+0x0/0x3f0 [spl] [] ? kthread+0x96/0xa0 [] ? child_rip+0xa/0x20 [] ? kthread+0x0/0xa0 [] ? child_rip+0x0/0x20 Mem-Info: Node 0 DMA per-cpu: CPU 0: hi: 0, btch: 1 usd: 0 Node 0 DMA32 per-cpu: CPU 0: hi: 186, btch: 31 usd: 0 active_anon:0 inactive_anon:0 isolated_anon:0 active_file:11 inactive_file:0 isolated_file:0 unevictable:0 dirty:0 writeback:0 unstable:0 free:8559 slab_reclaimable:1482 slab_unreclaimable:12252 mapped:1 shmem:0 pagetables:1242 bounce:0 Node 0 DMA free:8352kB min:332kB low:412kB high:496kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15348kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:20kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes lowmem_reserve[]: 0 2004 2004 2004 Node 0 DMA32 free:25884kB min:44720kB low:55900kB high:67080kB active_anon:0kB inactive_anon:0kB active_file:44kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:2052308kB mlocked:0kB dirty:0kB writeback:0kB mapped:4kB shmem:0kB slab_reclaimable:5928kB slab_unreclaimable:48988kB kernel_stack:3416kB pagetables:4968kB unstable:0kB bounce:0kB writeback_tmp:0kB pages_scanned:100 all_unreclaimable? yes lowmem_reserve[]: 0 0 0 0 Node 0 DMA: 0*4kB 0*8kB 2*16kB 0*32kB 0*64kB 1*128kB 0*256kB 0*512kB 0*1024kB 2*2048kB 1*4096kB = 8352kB Node 0 DMA32: 719*4kB 340*8kB 184*16kB 84*32kB 33*64kB 16*128kB 5*256kB 2*512kB 2*1024kB 1*2048kB 1*4096kB = 25884kB 20 total pagecache pages 0 pages in swap cache Swap cache stats: add 5121, delete 5121, find 16/25 Free swap = 4108600kB Total swap = 4128764kB 524284 pages RAM 43654 pages reserved 54 pages shared 465254 pages non-shared [ pid ] uid tgid total_vm rss cpu oom_adj oom_score_adj name [ 352] 0 352 2762 0 0 -17 -1000 udevd [ 682] 0 682 2761 0 0 -17 -1000 udevd [ 1104] 0 1104 2280 1 0 0 0 dhclient [ 1156] 0 1156 6910 1 0 -17 -1000 auditd [ 1172] 0 1172 63854 1 0 0 0 rsyslogd [ 1212] 32 1212 4744 1 0 0 0 rpcbind [ 1224] 0 1224 49916 1 0 0 0 sssd [ 1225] 0 1225 64219 1 0 0 0 sssd_be [ 1226] 0 1226 50377 1 0 0 0 sssd_nss [ 1227] 0 1227 47954 1 0 0 0 sssd_pam [ 1228] 0 1228 47403 1 0 0 0 sssd_ssh [ 1229] 0 1229 52533 1 0 0 0 sssd_pac [ 1246] 29 1246 5837 1 0 0 0 rpc.statd [ 6863] 81 6863 5871 1 0 0 0 dbus-daemon [ 6901] 0 6901 1020 0 0 0 0 acpid [ 6910] 68 6910 9927 1 0 0 0 hald [ 6911] 0 6911 5082 1 0 0 0 hald-runner [ 6943] 0 6943 5612 1 0 0 0 hald-addon-inpu [ 6957] 68 6957 4484 1 0 0 0 hald-addon-acpi [ 6974] 0 6974 151441 1 0 0 0 automount [ 7015] 0 7015 26827 0 0 0 0 rpc.rquotad [ 7019] 0 7019 5414 0 0 0 0 rpc.mountd [ 7055] 0 7055 5773 1 0 0 0 rpc.idmapd [ 7086] 496 7086 56787 1 0 0 0 munged [ 7101] 0 7101 16656 0 0 -17 -1000 sshd [ 7109] 0 7109 5545 1 0 0 0 xinetd [ 7117] 38 7117 8205 1 0 0 0 ntpd [ 7193] 0 7193 20846 1 0 0 0 master [ 7206] 89 7206 20909 1 0 0 0 qmgr [ 7216] 0 7216 29324 1 0 0 0 crond [ 7227] 0 7227 5385 0 0 0 0 atd [ 7255] 0 7255 15590 1 0 0 0 certmonger [ 7267] 0 7267 1016 1 0 0 0 mingetty [ 7269] 0 7269 1016 1 0 0 0 mingetty [ 7271] 0 7271 1016 1 0 0 0 mingetty [ 7273] 0 7273 1020 1 0 0 0 agetty [ 7274] 0 7274 1016 1 0 0 0 mingetty [ 7276] 0 7276 1016 1 0 0 0 mingetty [ 7278] 0 7278 1016 1 0 0 0 mingetty [ 6377] 89 6377 20866 1 0 0 0 pickup Kernel panic - not syncing: Out of memory: system-wide panic_on_oom is enabled Pid: 396, comm: spl_kmem_cache/ Tainted: P --------------- 2.6.32-431.29.2.el6_lustre.g9835a2a.x86_64 #1 Call Trace: [] ? panic+0xa7/0x16f [] ? dump_header+0x191/0x1b0 [] ? check_panic_on_oom+0x7c/0x80 [] ? out_of_memory+0x1bb/0x3c0 [] ? __alloc_pages_nodemask+0x89f/0x8d0 [] ? alloc_pages_current+0xaa/0x110 [] ? __get_free_pages+0xe/0x50 [] ? pte_alloc_one_kernel+0x15/0x20 [] ? __pte_alloc_kernel+0x1b/0xc0 [] ? vmap_page_range_noflush+0x309/0x370 [] ? map_vm_area+0x32/0x50 [] ? __vmalloc_area_node+0x100/0x190 [] ? kv_alloc+0x59/0x60 [spl] [] ? __vmalloc_node+0xad/0x120 [] ? kv_alloc+0x59/0x60 [spl] [] ? __vmalloc+0x22/0x30 [] ? kv_alloc+0x59/0x60 [spl] [] ? spl_cache_grow_work+0x39/0x2d0 [spl] [] ? __wake_up+0x53/0x70 [] ? taskq_thread+0x1e7/0x3f0 [spl] [] ? default_wake_function+0x0/0x20 [] ? taskq_thread+0x0/0x3f0 [spl] [] ? kthread+0x96/0xa0 [] ? child_rip+0xa/0x20 [] ? kthread+0x0/0xa0 [] ? child_rip+0x0/0x20