[LU-6324] CLASSERT(sizeof(struct osd_thread_info) <= PAGE_SIZE) fails for some configs Created: 03/Mar/15  Updated: 07/Jul/15  Resolved: 07/Jul/15

Status: Resolved
Project: Lustre
Component/s: None
Affects Version/s: Lustre 2.8.0
Fix Version/s: Lustre 2.8.0

Type: Bug Priority: Major
Reporter: John Hammond Assignee: Alex Zhuravlev
Resolution: Fixed Votes: 0
Labels: None

Attachments: File config-2.6.32-431.29.2.el6.lustre.x86_64    
Issue Links:
Related
is related to LU-3680 OOM crash: null_alloc_rs()) ASSERTION... Resolved
Severity: 3
Rank (Obsolete): 17696

 Description   

I am unable to build v2_7_50_0-10-g56875fd (LU-3680 osd: reduce osd_thread_info in ldiskfs osd, http://review.whamcloud.com/9726) due to the added CLASSERT() in osd_mod_init() and the that fact that I enable several CONFIG_DEBUG_*.

struct osd_thread_info {
        const struct lu_env  *     oti_env;              /*     0     8 */
        struct dentry              oti_obj_dentry;       /*     8   256 */
        /* --- cacheline 4 boundary (256 bytes) was 8 bytes ago --- */
        struct dentry              oti_child_dentry;     /*   264   256 */
        /* --- cacheline 8 boundary (512 bytes) was 8 bytes ago --- */
        struct dentry              oti_it_dentry;        /*   520   256 */
        /* --- cacheline 12 boundary (768 bytes) was 8 bytes ago --- */
        union {
                struct file        oti_file;             /*         304 */
                struct kstatfs     oti_ksfs;             /*         120 */
        };                                               /*   776   304 */
        /* --- cacheline 16 boundary (1024 bytes) was 56 bytes ago --- */
        struct htree_lock *        oti_hlock;            /*  1080     8 */
        /* --- cacheline 17 boundary (1088 bytes) --- */
        struct lu_fid              oti_fid;              /*  1088    16 */
        struct lu_fid              oti_fid2;             /*  1104    16 */
        struct lu_fid              oti_fid3;             /*  1120    16 */
        struct osd_inode_id        oti_id;               /*  1136     8 */
        struct osd_inode_id        oti_id2;              /*  1144     8 */
        /* --- cacheline 18 boundary (1152 bytes) --- */
        struct osd_inode_id        oti_id3;              /*  1152     8 */
        struct ost_id              oti_ostid;            /*  1160    16 */
        struct timespec            oti_time;             /*  1176    16 */
        struct lustre_capa_key     oti_capa_key;         /*  1192    72 */
        /* --- cacheline 19 boundary (1216 bytes) was 48 bytes ago --- */
        struct lustre_capa         oti_capa;             /*  1264   120 */
        /* --- cacheline 21 boundary (1344 bytes) was 40 bytes ago --- */
        struct osd_device *        oti_dev;              /*  1384     8 */
        void *                     oti_it_ea_buf;        /*  1392     8 */
        unsigned int               oti_it_ea_buf_used:1; /*  1400:31  4 */

        /* XXX 31 bits hole, try to pack */
        /* XXX 4 bytes hole, try to pack */

        /* --- cacheline 22 boundary (1408 bytes) --- */
        struct iam_iterator        oti_idx_it;           /*  1408   312 */
        /* --- cacheline 26 boundary (1664 bytes) was 56 bytes ago --- */
        union {
                char               oti_name[48];         /*          48 */
                char               oti_it_ipd[816];      /*         816 */
                long long int      oti_alignment_lieutenant; /*           8 */
        };                                               /*  1720   816 */
        /* --- cacheline 39 boundary (2496 bytes) was 40 bytes ago --- */
        union {
                char               oti_idx_ipd[816];     /*         816 */
                long long int      oti_alignment_lieutenant_colonel; /*           8 */
        };                                               /*  2536   816 */
        /* --- cacheline 52 boundary (3328 bytes) was 24 bytes ago --- */
        struct osd_idmap_cache     oti_cache;            /*  3352    32 */
        int                        oti_r_locks;          /*  3384     4 */
        int                        oti_w_locks;          /*  3388     4 */
        /* --- cacheline 53 boundary (3392 bytes) --- */
        int                        oti_txns;             /*  3392     4 */

        /* XXX 4 bytes hole, try to pack */

        struct lu_buf              oti_buf;              /*  3400    16 */
        struct lu_buf              oti_big_buf;          /*  3416    16 */
        union {
                struct lustre_mdt_attrs oti_mdt_attrs;   /*          24 */
                char               oti_mdt_attrs_old[64]; /*          64 */
                struct filter_fid_old oti_ff;            /*          32 */
                struct filter_fid  oti_ff_new;           /*          16 */
        };                                               /*  3432    64 */
        /* --- cacheline 54 boundary (3456 bytes) was 40 bytes ago --- */
        struct osd_iobuf           oti_iobuf;            /*  3496   184 */

        /* XXX last struct has 4 bytes of padding */

        /* --- cacheline 57 boundary (3648 bytes) was 32 bytes ago --- */
        struct inode *             oti_inode;            /*  3680     8 */
        char                       oti_ldp[32];          /*  3688    32 */
        /* --- cacheline 58 boundary (3712 bytes) was 8 bytes ago --- */
        char                       oti_ldp2[32];         /*  3720    32 */
        union {
                struct if_dqblk    oti_dqblk;            /*          72 */
                struct if_dqinfo   oti_dqinfo;           /*          24 */
        };                                               /*  3752    72 */
        /* --- cacheline 59 boundary (3776 bytes) was 48 bytes ago --- */
        struct lquota_id_info      oti_qi;               /*  3824    48 */

        /* XXX last struct has 7 bytes of padding */

        /* --- cacheline 60 boundary (3840 bytes) was 32 bytes ago --- */
        struct lquota_trans        oti_quota_trans;      /*  3872   200 */
        /* --- cacheline 63 boundary (4032 bytes) was 40 bytes ago --- */
        union lquota_rec           oti_quota_rec;        /*  4072    32 */
        /* --- cacheline 64 boundary (4096 bytes) was 8 bytes ago --- */
        __u64                      oti_quota_id;         /*  4104     8 */
        struct lu_seq_range        oti_seq_range;        /*  4112    24 */
        short unsigned int         oti_declare_ops[11];  /*  4136    22 */
        short unsigned int         oti_declare_ops_rb[11]; /*  4158    22 */
        /* --- cacheline 65 boundary (4160 bytes) was 20 bytes ago --- */
        short unsigned int         oti_declare_ops_cred[11]; /*  4180    22 */
        bool                       oti_rollback;         /*  4202     1 */

        /* size: 4208, cachelines: 66, members: 43 */
        /* sum members: 4195, holes: 2, sum holes: 8 */
        /* bit holes: 1, sum bit holes: 31 bits */
        /* padding: 5 */
        /* paddings: 2, sum paddings: 11 */
        /* last cacheline: 48 bytes */
};

My config is attached.

 Comments   
Comment by Andreas Dilger [ 03/Mar/15 ]

It seems unlikely that the struct can shrink by another 112 bytes. It might make sense to find the CONFIG option(s that John has enabled that makes the struct so large compared to the default config from RHEL and put the CLASSERT under #ifndef so that it doesn't trip in that case.

Comment by Alex Zhuravlev [ 03/Mar/15 ]

Andreas, that's possible with http://review.whamcloud.com/#/c/5572/ (capa support removal):

(gdb) p sizeof(struct lustre_capa_key)
$1 = 72
(gdb) p sizeof(struct lustre_capa)
$2 = 120

Comment by Alex Zhuravlev [ 03/Mar/15 ]

though I'm fine to disable the check like you suggested.

Comment by John Hammond [ 03/Mar/15 ]

It could also be converted to WARN_ON().

Comment by Alex Zhuravlev [ 03/Mar/15 ]

John, would you mind to put CLASSERT() under #ifndef CONFIG_DEBUG_MUTEXES and #ifndef CONFIG_DEBUG_SPINLOCK please ?

Comment by Gerrit Updater [ 03/Mar/15 ]

John L. Hammond (john.hammond@intel.com) uploaded a new patch: http://review.whamcloud.com/13955
Subject: LU-6324 osd: allow larger osd_thread_info for debug
Project: fs/lustre-release
Branch: master
Current Patch Set: 1
Commit: 9d2a721fa5289f788f9c3c718752abe5fb1d8517

Comment by Oleg Drokin [ 03/Mar/15 ]

This is not the real reason for blown up structure sizes.

I have all of those set and my struct size is still about 3500

[green@intelbox lustre-release]$ grep CONFIG_DEBUG_MUTEXES ~/bk/linux-2.6.32-504.3.3.el6-debug/.config
CONFIG_DEBUG_MUTEXES=y
[green@intelbox lustre-release]$ grep CONFIG_DEBUG_SPINLOCK ~/bk/linux-2.6.32-504.3.3.el6-debug/.config
CONFIG_DEBUG_SPINLOCK=y
CONFIG_DEBUG_SPINLOCK_SLEEP=y
Reading symbols from /home/green/git/lustre-release/lustre/osd-ldiskfs/osd_ldiskfs.ko...done.
(gdb) p sizeof(struct osd_thread_info)
$1 = 3576

The culprit must be something else

Comment by John Hammond [ 03/Mar/15 ]

It works for me.

Comment by Oleg Drokin [ 03/Mar/15 ]

that's because you happen to have those two set too.

I guess they are a good proxy for "debug enabled kernel" in the end but it might be interesting to see what is the real reason.

here's a relevant diff between my (orig) and your configs:

 #
 # Kernel hacking
@@ -4111,7 +4227,7 @@
 # CONFIG_UNUSED_SYMBOLS is not set
 CONFIG_DEBUG_FS=y
 CONFIG_HEADERS_CHECK=y
-# CONFIG_DEBUG_SECTION_MISMATCH is not set
+CONFIG_DEBUG_SECTION_MISMATCH=y
 CONFIG_DEBUG_KERNEL=y
 CONFIG_DEBUG_SHIRQ=y
 CONFIG_DETECT_SOFTLOCKUP=y
@@ -4119,8 +4235,8 @@
 CONFIG_HARDLOCKUP_DETECTOR=y
 CONFIG_BOOTPARAM_HARDLOCKUP_ENABLED=y
 CONFIG_BOOTPARAM_HARDLOCKUP_ENABLED_VALUE=1
-# CONFIG_BOOTPARAM_HARDLOCKUP_PANIC is not set
-CONFIG_BOOTPARAM_HARDLOCKUP_PANIC_VALUE=0
+CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y
+CONFIG_BOOTPARAM_HARDLOCKUP_PANIC_VALUE=1
 # CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC is not set
 CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE=0
 CONFIG_DETECT_HUNG_TASK=y
@@ -4132,33 +4248,39 @@
 CONFIG_TIMER_STATS=y
 CONFIG_DEBUG_OBJECTS=y
 # CONFIG_DEBUG_OBJECTS_SELFTEST is not set
-CONFIG_DEBUG_OBJECTS_FREE=y
+# CONFIG_DEBUG_OBJECTS_FREE is not set
 # CONFIG_DEBUG_OBJECTS_TIMERS is not set
 CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT=1
 CONFIG_DEBUG_SLAB=y
 # CONFIG_DEBUG_SLAB_LEAK is not set
-# CONFIG_DEBUG_KMEMLEAK is not set
-# CONFIG_DEBUG_RT_MUTEXES is not set
+CONFIG_DEBUG_KMEMLEAK=y
+CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE=400
+# CONFIG_DEBUG_KMEMLEAK_TEST is not set
+# CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF is not set
+CONFIG_DEBUG_RT_MUTEXES=y
+CONFIG_DEBUG_PI_LIST=y
 # CONFIG_RT_MUTEX_TESTER is not set
 CONFIG_DEBUG_SPINLOCK=y
 CONFIG_DEBUG_MUTEXES=y
-# CONFIG_DEBUG_WW_MUTEX_SLOWPATH is not set
-# CONFIG_DEBUG_LOCK_ALLOC is not set
-# CONFIG_PROVE_LOCKING is not set
-# CONFIG_LOCK_STAT is not set
+CONFIG_DEBUG_LOCK_ALLOC=y
+CONFIG_PROVE_LOCKING=y
+CONFIG_LOCKDEP=y
+CONFIG_LOCK_STAT=y
+# CONFIG_DEBUG_LOCKDEP is not set
+CONFIG_TRACE_IRQFLAGS=y
 CONFIG_DEBUG_SPINLOCK_SLEEP=y
 # CONFIG_DEBUG_LOCKING_API_SELFTESTS is not set
 CONFIG_STACKTRACE=y
 # CONFIG_DEBUG_KOBJECT is not set
 CONFIG_DEBUG_BUGVERBOSE=y
 CONFIG_DEBUG_INFO=y
-# CONFIG_DEBUG_VM is not set
-# CONFIG_DEBUG_VIRTUAL is not set
-CONFIG_DEBUG_WRITECOUNT=y
+CONFIG_DEBUG_VM=y
+CONFIG_DEBUG_VIRTUAL=y
+# CONFIG_DEBUG_WRITECOUNT is not set
 CONFIG_DEBUG_MEMORY_INIT=y
 CONFIG_DEBUG_LIST=y
 # CONFIG_DEBUG_SG is not set
-CONFIG_DEBUG_NOTIFIERS=y
+# CONFIG_DEBUG_NOTIFIERS is not set
 # CONFIG_DEBUG_CREDENTIALS is not set
 CONFIG_ARCH_WANT_FRAME_POINTERS=y
 CONFIG_FRAME_POINTER=y
@@ -4170,15 +4292,10 @@
 # CONFIG_DEBUG_BLOCK_EXT_DEVT is not set
 # CONFIG_DEBUG_FORCE_WEAK_PER_CPU is not set
 # CONFIG_LKDTM is not set
-CONFIG_FAULT_INJECTION=y
-CONFIG_FAILSLAB=y
-CONFIG_FAIL_PAGE_ALLOC=y
-CONFIG_FAIL_MAKE_REQUEST=y
-# CONFIG_FAIL_IO_TIMEOUT is not set
-# CONFIG_FAULT_INJECTION_DEBUG_FS is not set
-CONFIG_LATENCYTOP=y
-CONFIG_SYSCTL_SYSCALL_CHECK=y
-CONFIG_DEBUG_PAGEALLOC=y
+# CONFIG_FAULT_INJECTION is not set
+# CONFIG_LATENCYTOP is not set
+# CONFIG_SYSCTL_SYSCALL_CHECK is not set
+# CONFIG_DEBUG_PAGEALLOC is not set
 CONFIG_USER_STACKTRACE_SUPPORT=y
 CONFIG_NOP_TRACER=y
 CONFIG_HAVE_FTRACE_NMI_ENTER=y
@@ -4221,12 +4338,12 @@
 # CONFIG_FTRACE_STARTUP_TEST is not set
 # CONFIG_MMIOTRACE is not set
 CONFIG_RING_BUFFER_BENCHMARK=m
-# CONFIG_PROVIDE_OHCI1394_DMA_INIT is not set
+CONFIG_PROVIDE_OHCI1394_DMA_INIT=y
 # CONFIG_FIREWIRE_OHCI_REMOTE_DMA is not set
-# CONFIG_BUILD_DOCSRC is not set
+CONFIG_BUILD_DOCSRC=y
 CONFIG_DYNAMIC_DEBUG=y
 # CONFIG_DMA_API_DEBUG is not set
-# CONFIG_ASYNC_RAID6_TEST is not set
+CONFIG_ASYNC_RAID6_TEST=m
 # CONFIG_SAMPLES is not set
 CONFIG_HAVE_ARCH_KGDB=y
 CONFIG_KGDB=y
@@ -4235,18 +4352,17 @@
 # CONFIG_KGDB_TESTS_ON_BOOT is not set
 CONFIG_HAVE_ARCH_KMEMCHECK=y
 # CONFIG_TEST_KSTRTOX is not set
-# CONFIG_STRICT_DEVMEM is not set
+CONFIG_STRICT_DEVMEM=y
 # CONFIG_X86_VERBOSE_BOOTUP is not set
 CONFIG_EARLY_PRINTK=y
-# CONFIG_EARLY_PRINTK_DBGP is not set
-# CONFIG_EARLY_PRINTK_EFI is not set
+CONFIG_EARLY_PRINTK_DBGP=y
 CONFIG_DEBUG_STACKOVERFLOW=y
-CONFIG_DEBUG_STACK_USAGE=y
+# CONFIG_DEBUG_STACK_USAGE is not set
 # CONFIG_DEBUG_PER_CPU_MAPS is not set
 # CONFIG_X86_PTDUMP is not set
 CONFIG_DEBUG_RODATA=y
-# CONFIG_DEBUG_RODATA_TEST is not set
-# CONFIG_DEBUG_NX_TEST is not set
+CONFIG_DEBUG_RODATA_TEST=y
+CONFIG_DEBUG_NX_TEST=m
 # CONFIG_IOMMU_DEBUG is not set
 # CONFIG_IOMMU_STRESS is not set
 CONFIG_HAVE_MMIOTRACE_SUPPORT=y
Comment by John Hammond [ 04/Mar/15 ]

Oleg, can you post the output of 'pahole lustre/osd-ldiskfs/osd_handler.o'?

Comment by Gerrit Updater [ 11/Mar/15 ]

Oleg Drokin (oleg.drokin@intel.com) merged in patch http://review.whamcloud.com/13955/
Subject: LU-6324 osd: allow larger osd_thread_info for debug
Project: fs/lustre-release
Branch: master
Current Patch Set:
Commit: c65d83dd93fc61c44a1f0503e196a123d57dfb88

Generated at Sat Feb 10 01:59:14 UTC 2024 using Jira 9.4.14#940014-sha1:734e6822bbf0d45eff9af51f82432957f73aa32c.