[XFS] Keep stack usage down for 4k stacks by using noinline.
gcc-4.1 and more recent aggressively inline static functions which
increases XFS stack usage by ~15% in critical paths. Prevent this from
occurring by adding noinline to the STATIC definition.
Also uninline some functions that are too large to be inlined and were
causing problems with CONFIG_FORCED_INLINING=y.
Finally, clean up all the different users of inline, __inline and
__inline__ and put them under one STATIC_INLINE macro. For debug kernels
the STATIC_INLINE macro uninlines those functions.
SGI-PV: 957159
SGI-Modid: xfs-linux-melb:xfs-kern:27585a
Signed-off-by: David Chinner <dgc@sgi.com>
Signed-off-by: David Chatterton <chatz@sgi.com>
Signed-off-by: Tim Shimmin <tes@sgi.com>
diff --git a/fs/xfs/linux-2.6/xfs_aops.c b/fs/xfs/linux-2.6/xfs_aops.c
index 7b54461..9761a5b 100644
--- a/fs/xfs/linux-2.6/xfs_aops.c
+++ b/fs/xfs/linux-2.6/xfs_aops.c
@@ -249,7 +249,7 @@
return -error;
}
-STATIC inline int
+STATIC_INLINE int
xfs_iomap_valid(
xfs_iomap_t *iomapp,
loff_t offset)
diff --git a/fs/xfs/linux-2.6/xfs_buf.c b/fs/xfs/linux-2.6/xfs_buf.c
index 168eecd..e2bea6a 100644
--- a/fs/xfs/linux-2.6/xfs_buf.c
+++ b/fs/xfs/linux-2.6/xfs_buf.c
@@ -34,13 +34,13 @@
#include <linux/backing-dev.h>
#include <linux/freezer.h>
-STATIC kmem_zone_t *xfs_buf_zone;
-STATIC kmem_shaker_t xfs_buf_shake;
+static kmem_zone_t *xfs_buf_zone;
+static kmem_shaker_t xfs_buf_shake;
STATIC int xfsbufd(void *);
STATIC int xfsbufd_wakeup(int, gfp_t);
STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
-STATIC struct workqueue_struct *xfslogd_workqueue;
+static struct workqueue_struct *xfslogd_workqueue;
struct workqueue_struct *xfsdatad_workqueue;
#ifdef XFS_BUF_TRACE
@@ -139,7 +139,7 @@
return mask;
}
-STATIC inline void
+STATIC_INLINE void
set_page_region(
struct page *page,
size_t offset,
@@ -151,7 +151,7 @@
SetPageUptodate(page);
}
-STATIC inline int
+STATIC_INLINE int
test_page_region(
struct page *page,
size_t offset,
@@ -171,9 +171,9 @@
struct a_list *next;
} a_list_t;
-STATIC a_list_t *as_free_head;
-STATIC int as_list_len;
-STATIC DEFINE_SPINLOCK(as_lock);
+static a_list_t *as_free_head;
+static int as_list_len;
+static DEFINE_SPINLOCK(as_lock);
/*
* Try to batch vunmaps because they are costly.
@@ -1085,7 +1085,7 @@
return status;
}
-STATIC __inline__ int
+STATIC_INLINE int
_xfs_buf_iolocked(
xfs_buf_t *bp)
{
@@ -1095,7 +1095,7 @@
return 0;
}
-STATIC __inline__ void
+STATIC_INLINE void
_xfs_buf_ioend(
xfs_buf_t *bp,
int schedule)
@@ -1426,8 +1426,8 @@
/*
* buftarg list for delwrite queue processing
*/
-STATIC LIST_HEAD(xfs_buftarg_list);
-STATIC DEFINE_SPINLOCK(xfs_buftarg_lock);
+LIST_HEAD(xfs_buftarg_list);
+static DEFINE_SPINLOCK(xfs_buftarg_lock);
STATIC void
xfs_register_buftarg(
diff --git a/fs/xfs/linux-2.6/xfs_export.c b/fs/xfs/linux-2.6/xfs_export.c
index 5fb75d9..e3a5fed 100644
--- a/fs/xfs/linux-2.6/xfs_export.c
+++ b/fs/xfs/linux-2.6/xfs_export.c
@@ -24,7 +24,7 @@
#include "xfs_mount.h"
#include "xfs_export.h"
-STATIC struct dentry dotdot = { .d_name.name = "..", .d_name.len = 2, };
+static struct dentry dotdot = { .d_name.name = "..", .d_name.len = 2, };
/*
* XFS encodes and decodes the fileid portion of NFS filehandles
diff --git a/fs/xfs/linux-2.6/xfs_file.c b/fs/xfs/linux-2.6/xfs_file.c
index d26f5cd..cb51dc9 100644
--- a/fs/xfs/linux-2.6/xfs_file.c
+++ b/fs/xfs/linux-2.6/xfs_file.c
@@ -46,7 +46,7 @@
static struct vm_operations_struct xfs_dmapi_file_vm_ops;
#endif
-STATIC inline ssize_t
+STATIC_INLINE ssize_t
__xfs_file_read(
struct kiocb *iocb,
const struct iovec *iov,
@@ -84,7 +84,7 @@
return __xfs_file_read(iocb, iov, nr_segs, IO_ISAIO|IO_INVIS, pos);
}
-STATIC inline ssize_t
+STATIC_INLINE ssize_t
__xfs_file_write(
struct kiocb *iocb,
const struct iovec *iov,
diff --git a/fs/xfs/linux-2.6/xfs_iops.c b/fs/xfs/linux-2.6/xfs_iops.c
index 3ba814a..4dd6cb5 100644
--- a/fs/xfs/linux-2.6/xfs_iops.c
+++ b/fs/xfs/linux-2.6/xfs_iops.c
@@ -250,13 +250,13 @@
*
* XXX(hch): nfsd is broken, better fix it instead.
*/
-STATIC inline int
+STATIC_INLINE int
xfs_has_fs_struct(struct task_struct *task)
{
return (task->fs != init_task.fs);
}
-STATIC inline void
+STATIC void
xfs_cleanup_inode(
bhv_vnode_t *dvp,
bhv_vnode_t *vp,
diff --git a/fs/xfs/linux-2.6/xfs_super.c b/fs/xfs/linux-2.6/xfs_super.c
index b93265b..dab5d38 100644
--- a/fs/xfs/linux-2.6/xfs_super.c
+++ b/fs/xfs/linux-2.6/xfs_super.c
@@ -58,10 +58,10 @@
#include <linux/kthread.h>
#include <linux/freezer.h>
-STATIC struct quotactl_ops xfs_quotactl_operations;
-STATIC struct super_operations xfs_super_operations;
-STATIC kmem_zone_t *xfs_vnode_zone;
-STATIC kmem_zone_t *xfs_ioend_zone;
+static struct quotactl_ops xfs_quotactl_operations;
+static struct super_operations xfs_super_operations;
+static kmem_zone_t *xfs_vnode_zone;
+static kmem_zone_t *xfs_ioend_zone;
mempool_t *xfs_ioend_pool;
STATIC struct xfs_mount_args *
@@ -121,7 +121,7 @@
return (((__uint64_t)pagefactor) << bitshift) - 1;
}
-STATIC __inline__ void
+STATIC_INLINE void
xfs_set_inodeops(
struct inode *inode)
{
@@ -147,7 +147,7 @@
}
}
-STATIC __inline__ void
+STATIC_INLINE void
xfs_revalidate_inode(
xfs_mount_t *mp,
bhv_vnode_t *vp,
@@ -873,7 +873,7 @@
mnt);
}
-STATIC struct super_operations xfs_super_operations = {
+static struct super_operations xfs_super_operations = {
.alloc_inode = xfs_fs_alloc_inode,
.destroy_inode = xfs_fs_destroy_inode,
.write_inode = xfs_fs_write_inode,
@@ -887,7 +887,7 @@
.show_options = xfs_fs_show_options,
};
-STATIC struct quotactl_ops xfs_quotactl_operations = {
+static struct quotactl_ops xfs_quotactl_operations = {
.quota_sync = xfs_fs_quotasync,
.get_xstate = xfs_fs_getxstate,
.set_xstate = xfs_fs_setxstate,
diff --git a/fs/xfs/linux-2.6/xfs_sysctl.c b/fs/xfs/linux-2.6/xfs_sysctl.c
index af24653..b1036dd 100644
--- a/fs/xfs/linux-2.6/xfs_sysctl.c
+++ b/fs/xfs/linux-2.6/xfs_sysctl.c
@@ -54,7 +54,7 @@
}
#endif /* CONFIG_PROC_FS */
-STATIC ctl_table xfs_table[] = {
+static ctl_table xfs_table[] = {
{XFS_RESTRICT_CHOWN, "restrict_chown", &xfs_params.restrict_chown.val,
sizeof(int), 0644, NULL, &proc_dointvec_minmax,
&sysctl_intvec, NULL,
@@ -136,12 +136,12 @@
{0}
};
-STATIC ctl_table xfs_dir_table[] = {
+static ctl_table xfs_dir_table[] = {
{FS_XFS, "xfs", NULL, 0, 0555, xfs_table},
{0}
};
-STATIC ctl_table xfs_root_table[] = {
+static ctl_table xfs_root_table[] = {
{CTL_FS, "fs", NULL, 0, 0555, xfs_dir_table},
{0}
};
diff --git a/fs/xfs/linux-2.6/xfs_vnode.c b/fs/xfs/linux-2.6/xfs_vnode.c
index 553fa73..ada24ba 100644
--- a/fs/xfs/linux-2.6/xfs_vnode.c
+++ b/fs/xfs/linux-2.6/xfs_vnode.c
@@ -26,7 +26,7 @@
*/
#define NVSYNC 37
#define vptosync(v) (&vsync[((unsigned long)v) % NVSYNC])
-STATIC wait_queue_head_t vsync[NVSYNC];
+static wait_queue_head_t vsync[NVSYNC];
void
vn_init(void)
diff --git a/fs/xfs/linux-2.6/xfs_vnode.h b/fs/xfs/linux-2.6/xfs_vnode.h
index 515f5fd..b76118c 100644
--- a/fs/xfs/linux-2.6/xfs_vnode.h
+++ b/fs/xfs/linux-2.6/xfs_vnode.h
@@ -489,14 +489,14 @@
#define VN_LOCK(vp) mutex_spinlock(&(vp)->v_lock)
#define VN_UNLOCK(vp, s) mutex_spinunlock(&(vp)->v_lock, s)
-static __inline__ void vn_flagset(struct bhv_vnode *vp, uint flag)
+STATIC_INLINE void vn_flagset(struct bhv_vnode *vp, uint flag)
{
spin_lock(&vp->v_lock);
vp->v_flag |= flag;
spin_unlock(&vp->v_lock);
}
-static __inline__ uint vn_flagclr(struct bhv_vnode *vp, uint flag)
+STATIC_INLINE uint vn_flagclr(struct bhv_vnode *vp, uint flag)
{
uint cleared;
diff --git a/fs/xfs/quota/xfs_dquot_item.c b/fs/xfs/quota/xfs_dquot_item.c
index 33ad5af..aa3c76a 100644
--- a/fs/xfs/quota/xfs_dquot_item.c
+++ b/fs/xfs/quota/xfs_dquot_item.c
@@ -399,7 +399,7 @@
/*
* This is the ops vector for dquots
*/
-STATIC struct xfs_item_ops xfs_dquot_item_ops = {
+static struct xfs_item_ops xfs_dquot_item_ops = {
.iop_size = (uint(*)(xfs_log_item_t*))xfs_qm_dquot_logitem_size,
.iop_format = (void(*)(xfs_log_item_t*, xfs_log_iovec_t*))
xfs_qm_dquot_logitem_format,
@@ -606,7 +606,7 @@
return;
}
-STATIC struct xfs_item_ops xfs_qm_qoffend_logitem_ops = {
+static struct xfs_item_ops xfs_qm_qoffend_logitem_ops = {
.iop_size = (uint(*)(xfs_log_item_t*))xfs_qm_qoff_logitem_size,
.iop_format = (void(*)(xfs_log_item_t*, xfs_log_iovec_t*))
xfs_qm_qoff_logitem_format,
@@ -628,7 +628,7 @@
/*
* This is the ops vector shared by all quotaoff-start log items.
*/
-STATIC struct xfs_item_ops xfs_qm_qoff_logitem_ops = {
+static struct xfs_item_ops xfs_qm_qoff_logitem_ops = {
.iop_size = (uint(*)(xfs_log_item_t*))xfs_qm_qoff_logitem_size,
.iop_format = (void(*)(xfs_log_item_t*, xfs_log_iovec_t*))
xfs_qm_qoff_logitem_format,
diff --git a/fs/xfs/quota/xfs_qm.c b/fs/xfs/quota/xfs_qm.c
index 7c6a3a5..db1be5d 100644
--- a/fs/xfs/quota/xfs_qm.c
+++ b/fs/xfs/quota/xfs_qm.c
@@ -64,10 +64,10 @@
kmem_zone_t *qm_dqzone;
kmem_zone_t *qm_dqtrxzone;
-STATIC kmem_shaker_t xfs_qm_shaker;
+static kmem_shaker_t xfs_qm_shaker;
-STATIC cred_t xfs_zerocr;
-STATIC xfs_inode_t xfs_zeroino;
+static cred_t xfs_zerocr;
+static xfs_inode_t xfs_zeroino;
STATIC void xfs_qm_list_init(xfs_dqlist_t *, char *, int);
STATIC void xfs_qm_list_destroy(xfs_dqlist_t *);
diff --git a/fs/xfs/quota/xfs_qm_bhv.c b/fs/xfs/quota/xfs_qm_bhv.c
index db8872b..39a99ab 100644
--- a/fs/xfs/quota/xfs_qm_bhv.c
+++ b/fs/xfs/quota/xfs_qm_bhv.c
@@ -384,7 +384,7 @@
}
-STATIC struct xfs_qmops xfs_qmcore_xfs = {
+static struct xfs_qmops xfs_qmcore_xfs = {
.xfs_qminit = xfs_qm_newmount,
.xfs_qmdone = xfs_qm_unmount_quotadestroy,
.xfs_qmmount = xfs_qm_endmount,
diff --git a/fs/xfs/support/debug.h b/fs/xfs/support/debug.h
index 4f54dca..2a70cc6 100644
--- a/fs/xfs/support/debug.h
+++ b/fs/xfs/support/debug.h
@@ -38,13 +38,37 @@
#ifndef DEBUG
# define ASSERT(expr) ((void)0)
-#else
-# define ASSERT(expr) ASSERT_ALWAYS(expr)
-extern unsigned long random(void);
-#endif
#ifndef STATIC
-# define STATIC static
+# define STATIC static noinline
#endif
+#ifndef STATIC_INLINE
+# define STATIC_INLINE static inline
+#endif
+
+#else /* DEBUG */
+
+# define ASSERT(expr) ASSERT_ALWAYS(expr)
+extern unsigned long random(void);
+
+#ifndef STATIC
+# define STATIC noinline
+#endif
+
+/*
+ * We stop inlining of inline functions in debug mode.
+ * Unfortunately, this means static inline in header files
+ * get multiple definitions, so they need to remain static.
+ * This then gives tonnes of warnings about unused but defined
+ * functions, so we need to add the unused attribute to prevent
+ * these spurious warnings.
+ */
+#ifndef STATIC_INLINE
+# define STATIC_INLINE static __attribute__ ((unused)) noinline
+#endif
+
+#endif /* DEBUG */
+
+
#endif /* __XFS_SUPPORT_DEBUG_H__ */
diff --git a/fs/xfs/xfs_attr.c b/fs/xfs/xfs_attr.c
index 9ada7bd..c6c2596 100644
--- a/fs/xfs/xfs_attr.c
+++ b/fs/xfs/xfs_attr.c
@@ -57,9 +57,9 @@
*/
#define ATTR_SYSCOUNT 2
-STATIC struct attrnames posix_acl_access;
-STATIC struct attrnames posix_acl_default;
-STATIC struct attrnames *attr_system_names[ATTR_SYSCOUNT];
+static struct attrnames posix_acl_access;
+static struct attrnames posix_acl_default;
+static struct attrnames *attr_system_names[ATTR_SYSCOUNT];
/*========================================================================
* Function prototypes for the kernel.
@@ -2477,7 +2477,7 @@
return xfs_acl_vhasacl_default(vp);
}
-STATIC struct attrnames posix_acl_access = {
+static struct attrnames posix_acl_access = {
.attr_name = "posix_acl_access",
.attr_namelen = sizeof("posix_acl_access") - 1,
.attr_get = posix_acl_access_get,
@@ -2486,7 +2486,7 @@
.attr_exists = posix_acl_access_exists,
};
-STATIC struct attrnames posix_acl_default = {
+static struct attrnames posix_acl_default = {
.attr_name = "posix_acl_default",
.attr_namelen = sizeof("posix_acl_default") - 1,
.attr_get = posix_acl_default_get,
@@ -2495,7 +2495,7 @@
.attr_exists = posix_acl_default_exists,
};
-STATIC struct attrnames *attr_system_names[] =
+static struct attrnames *attr_system_names[] =
{ &posix_acl_access, &posix_acl_default };
diff --git a/fs/xfs/xfs_attr_leaf.c b/fs/xfs/xfs_attr_leaf.c
index 9719bbe..cc4ffa4 100644
--- a/fs/xfs/xfs_attr_leaf.c
+++ b/fs/xfs/xfs_attr_leaf.c
@@ -94,7 +94,7 @@
* Namespace helper routines
*========================================================================*/
-STATIC inline attrnames_t *
+STATIC_INLINE attrnames_t *
xfs_attr_flags_namesp(int flags)
{
return ((flags & XFS_ATTR_SECURE) ? &attr_secure:
@@ -105,7 +105,7 @@
* If namespace bits don't match return 0.
* If all match then return 1.
*/
-STATIC inline int
+STATIC_INLINE int
xfs_attr_namesp_match(int arg_flags, int ondisk_flags)
{
return XFS_ATTR_NSP_ONDISK(ondisk_flags) == XFS_ATTR_NSP_ARGS_TO_ONDISK(arg_flags);
@@ -116,7 +116,7 @@
* then return 0.
* If all match or are overridable then return 1.
*/
-STATIC inline int
+STATIC_INLINE int
xfs_attr_namesp_match_overrides(int arg_flags, int ondisk_flags)
{
if (((arg_flags & ATTR_SECURE) == 0) !=
diff --git a/fs/xfs/xfs_bit.c b/fs/xfs/xfs_bit.c
index 43be6a7..1afe07f 100644
--- a/fs/xfs/xfs_bit.c
+++ b/fs/xfs/xfs_bit.c
@@ -29,7 +29,7 @@
/*
* Index of high bit number in byte, -1 for none set, 0..7 otherwise.
*/
-STATIC const char xfs_highbit[256] = {
+static const char xfs_highbit[256] = {
-1, 0, 1, 1, 2, 2, 2, 2, /* 00 .. 07 */
3, 3, 3, 3, 3, 3, 3, 3, /* 08 .. 0f */
4, 4, 4, 4, 4, 4, 4, 4, /* 10 .. 17 */
diff --git a/fs/xfs/xfs_bmap_btree.c b/fs/xfs/xfs_bmap_btree.c
index a7b835b..3a92510 100644
--- a/fs/xfs/xfs_bmap_btree.c
+++ b/fs/xfs/xfs_bmap_btree.c
@@ -1862,7 +1862,7 @@
* xfs_bmbt_get_startblock, xfs_bmbt_get_blockcount and xfs_bmbt_get_state.
*/
-STATIC __inline__ void
+STATIC_INLINE void
__xfs_bmbt_get_all(
__uint64_t l0,
__uint64_t l1,
diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c
index 7a55c24..6c1bddc 100644
--- a/fs/xfs/xfs_buf_item.c
+++ b/fs/xfs/xfs_buf_item.c
@@ -660,7 +660,7 @@
/*
* This is the ops vector shared by all buf log items.
*/
-STATIC struct xfs_item_ops xfs_buf_item_ops = {
+static struct xfs_item_ops xfs_buf_item_ops = {
.iop_size = (uint(*)(xfs_log_item_t*))xfs_buf_item_size,
.iop_format = (void(*)(xfs_log_item_t*, xfs_log_iovec_t*))
xfs_buf_item_format,
diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c
index 6dba781..3b14427 100644
--- a/fs/xfs/xfs_extfree_item.c
+++ b/fs/xfs/xfs_extfree_item.c
@@ -227,7 +227,7 @@
/*
* This is the ops vector shared by all efi log items.
*/
-STATIC struct xfs_item_ops xfs_efi_item_ops = {
+static struct xfs_item_ops xfs_efi_item_ops = {
.iop_size = (uint(*)(xfs_log_item_t*))xfs_efi_item_size,
.iop_format = (void(*)(xfs_log_item_t*, xfs_log_iovec_t*))
xfs_efi_item_format,
@@ -525,7 +525,7 @@
/*
* This is the ops vector shared by all efd log items.
*/
-STATIC struct xfs_item_ops xfs_efd_item_ops = {
+static struct xfs_item_ops xfs_efd_item_ops = {
.iop_size = (uint(*)(xfs_log_item_t*))xfs_efd_item_size,
.iop_format = (void(*)(xfs_log_item_t*, xfs_log_iovec_t*))
xfs_efd_item_format,
diff --git a/fs/xfs/xfs_ialloc.c b/fs/xfs/xfs_ialloc.c
index a446e5a..b5feb3e 100644
--- a/fs/xfs/xfs_ialloc.c
+++ b/fs/xfs/xfs_ialloc.c
@@ -342,7 +342,7 @@
return 0;
}
-STATIC __inline xfs_agnumber_t
+STATIC_INLINE xfs_agnumber_t
xfs_ialloc_next_ag(
xfs_mount_t *mp)
{
diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c
index 44dfac5..cd51858 100644
--- a/fs/xfs/xfs_inode.c
+++ b/fs/xfs/xfs_inode.c
@@ -2125,7 +2125,7 @@
return 0;
}
-static __inline__ int xfs_inode_clean(xfs_inode_t *ip)
+STATIC_INLINE int xfs_inode_clean(xfs_inode_t *ip)
{
return (((ip->i_itemp == NULL) ||
!(ip->i_itemp->ili_format.ilf_fields & XFS_ILOG_ALL)) &&
diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c
index a7a9225..565d470 100644
--- a/fs/xfs/xfs_inode_item.c
+++ b/fs/xfs/xfs_inode_item.c
@@ -887,7 +887,7 @@
/*
* This is the ops vector shared by all buf log items.
*/
-STATIC struct xfs_item_ops xfs_inode_item_ops = {
+static struct xfs_item_ops xfs_inode_item_ops = {
.iop_size = (uint(*)(xfs_log_item_t*))xfs_inode_item_size,
.iop_format = (void(*)(xfs_log_item_t*, xfs_log_iovec_t*))
xfs_inode_item_format,
diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c
index 9dfae18..397730f 100644
--- a/fs/xfs/xfs_mount.c
+++ b/fs/xfs/xfs_mount.c
@@ -1802,7 +1802,7 @@
}
}
-STATIC inline void
+STATIC_INLINE void
xfs_icsb_lock_cntr(
xfs_icsb_cnts_t *icsbp)
{
@@ -1811,7 +1811,7 @@
}
}
-STATIC inline void
+STATIC_INLINE void
xfs_icsb_unlock_cntr(
xfs_icsb_cnts_t *icsbp)
{
@@ -1819,7 +1819,7 @@
}
-STATIC inline void
+STATIC_INLINE void
xfs_icsb_lock_all_counters(
xfs_mount_t *mp)
{
@@ -1832,7 +1832,7 @@
}
}
-STATIC inline void
+STATIC_INLINE void
xfs_icsb_unlock_all_counters(
xfs_mount_t *mp)
{