mirror of
https://github.com/torvalds/linux.git
synced 2026-03-07 23:04:33 +01:00
compiler-context-analysis: Remove __cond_lock() function-like helper
As discussed in [1], removing __cond_lock() will improve the readability of trylock code. Now that Sparse context tracking support has been removed, we can also remove __cond_lock(). Change existing APIs to either drop __cond_lock() completely, or make use of the __cond_acquires() function attribute instead. In particular, spinlock and rwlock implementations required switching over to inline helpers rather than statement-expressions for their trylock_* variants. Suggested-by: Peter Zijlstra <peterz@infradead.org> Signed-off-by: Marco Elver <elver@google.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://lore.kernel.org/all/20250207082832.GU7145@noisy.programming.kicks-ass.net/ [1] Link: https://patch.msgid.link/20251219154418.3592607-25-elver@google.com
This commit is contained in:
Marco Elver
Peter Zijlstra
parent
5b63d0ae94
commit
e4588c25c9
24 changed files with 163 additions and 195 deletions
|
|
@ -112,10 +112,8 @@ Keywords
|
|||
__releases_shared
|
||||
__acquire
|
||||
__release
|
||||
__cond_lock
|
||||
__acquire_shared
|
||||
__release_shared
|
||||
__cond_lock_shared
|
||||
__acquire_ret
|
||||
__acquire_shared_ret
|
||||
context_unsafe
|
||||
|
|
|
|||
|
|
@ -583,7 +583,7 @@ To access PTE-level page tables, a helper like :c:func:`!pte_offset_map_lock` or
|
|||
:c:func:`!pte_offset_map` can be used depending on stability requirements.
|
||||
These map the page table into kernel memory if required, take the RCU lock, and
|
||||
depending on variant, may also look up or acquire the PTE lock.
|
||||
See the comment on :c:func:`!__pte_offset_map_lock`.
|
||||
See the comment on :c:func:`!pte_offset_map_lock`.
|
||||
|
||||
Atomicity
|
||||
^^^^^^^^^
|
||||
|
|
@ -667,7 +667,7 @@ must be released via :c:func:`!pte_unmap_unlock`.
|
|||
.. note:: There are some variants on this, such as
|
||||
:c:func:`!pte_offset_map_rw_nolock` when we know we hold the PTE stable but
|
||||
for brevity we do not explore this. See the comment for
|
||||
:c:func:`!__pte_offset_map_lock` for more details.
|
||||
:c:func:`!pte_offset_map_lock` for more details.
|
||||
|
||||
When modifying data in ranges we typically only wish to allocate higher page
|
||||
tables as necessary, using these locks to avoid races or overwriting anything,
|
||||
|
|
@ -686,7 +686,7 @@ At the leaf page table, that is the PTE, we can't entirely rely on this pattern
|
|||
as we have separate PMD and PTE locks and a THP collapse for instance might have
|
||||
eliminated the PMD entry as well as the PTE from under us.
|
||||
|
||||
This is why :c:func:`!__pte_offset_map_lock` locklessly retrieves the PMD entry
|
||||
This is why :c:func:`!pte_offset_map_lock` locklessly retrieves the PMD entry
|
||||
for the PTE, carefully checking it is as expected, before acquiring the
|
||||
PTE-specific lock, and then *again* checking that the PMD entry is as expected.
|
||||
|
||||
|
|
|
|||
|
|
@ -548,11 +548,11 @@ int iwl_trans_read_config32(struct iwl_trans *trans, u32 ofs,
|
|||
return iwl_trans_pcie_read_config32(trans, ofs, val);
|
||||
}
|
||||
|
||||
bool _iwl_trans_grab_nic_access(struct iwl_trans *trans)
|
||||
bool iwl_trans_grab_nic_access(struct iwl_trans *trans)
|
||||
{
|
||||
return iwl_trans_pcie_grab_nic_access(trans);
|
||||
}
|
||||
IWL_EXPORT_SYMBOL(_iwl_trans_grab_nic_access);
|
||||
IWL_EXPORT_SYMBOL(iwl_trans_grab_nic_access);
|
||||
|
||||
void __releases(nic_access)
|
||||
iwl_trans_release_nic_access(struct iwl_trans *trans)
|
||||
|
|
|
|||
|
|
@ -1063,11 +1063,7 @@ int iwl_trans_sw_reset(struct iwl_trans *trans);
|
|||
void iwl_trans_set_bits_mask(struct iwl_trans *trans, u32 reg,
|
||||
u32 mask, u32 value);
|
||||
|
||||
bool _iwl_trans_grab_nic_access(struct iwl_trans *trans);
|
||||
|
||||
#define iwl_trans_grab_nic_access(trans) \
|
||||
__cond_lock(nic_access, \
|
||||
likely(_iwl_trans_grab_nic_access(trans)))
|
||||
bool iwl_trans_grab_nic_access(struct iwl_trans *trans);
|
||||
|
||||
void __releases(nic_access)
|
||||
iwl_trans_release_nic_access(struct iwl_trans *trans);
|
||||
|
|
|
|||
|
|
@ -553,10 +553,7 @@ void iwl_trans_pcie_free(struct iwl_trans *trans);
|
|||
void iwl_trans_pcie_free_pnvm_dram_regions(struct iwl_dram_regions *dram_regions,
|
||||
struct device *dev);
|
||||
|
||||
bool __iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans, bool silent);
|
||||
#define _iwl_trans_pcie_grab_nic_access(trans, silent) \
|
||||
__cond_lock(nic_access_nobh, \
|
||||
likely(__iwl_trans_pcie_grab_nic_access(trans, silent)))
|
||||
bool _iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans, bool silent);
|
||||
|
||||
void iwl_trans_pcie_check_product_reset_status(struct pci_dev *pdev);
|
||||
void iwl_trans_pcie_check_product_reset_mode(struct pci_dev *pdev);
|
||||
|
|
|
|||
|
|
@ -2327,7 +2327,7 @@ EXPORT_SYMBOL(iwl_trans_pcie_reset);
|
|||
* This version doesn't disable BHs but rather assumes they're
|
||||
* already disabled.
|
||||
*/
|
||||
bool __iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans, bool silent)
|
||||
bool _iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans, bool silent)
|
||||
{
|
||||
int ret;
|
||||
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
|
||||
|
|
@ -2415,7 +2415,7 @@ bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans)
|
|||
bool ret;
|
||||
|
||||
local_bh_disable();
|
||||
ret = __iwl_trans_pcie_grab_nic_access(trans, false);
|
||||
ret = _iwl_trans_pcie_grab_nic_access(trans, false);
|
||||
if (ret) {
|
||||
/* keep BHs disabled until iwl_trans_pcie_release_nic_access */
|
||||
return ret;
|
||||
|
|
|
|||
|
|
@ -341,24 +341,6 @@ static inline void _context_unsafe_alias(void **p) { }
|
|||
*/
|
||||
#define __release(x) __release_ctx_lock(x)
|
||||
|
||||
/**
|
||||
* __cond_lock() - function that conditionally acquires a context lock
|
||||
* exclusively
|
||||
* @x: context lock instance pinter
|
||||
* @c: boolean expression
|
||||
*
|
||||
* Return: result of @c
|
||||
*
|
||||
* No-op function that conditionally acquires context lock instance @x
|
||||
* exclusively, if the boolean expression @c is true. The result of @c is the
|
||||
* return value; for example:
|
||||
*
|
||||
* .. code-block:: c
|
||||
*
|
||||
* #define spin_trylock(l) __cond_lock(&lock, _spin_trylock(&lock))
|
||||
*/
|
||||
#define __cond_lock(x, c) __try_acquire_ctx_lock(x, c)
|
||||
|
||||
/**
|
||||
* __must_hold_shared() - function attribute, caller must hold shared context lock
|
||||
*
|
||||
|
|
@ -417,19 +399,6 @@ static inline void _context_unsafe_alias(void **p) { }
|
|||
*/
|
||||
#define __release_shared(x) __release_shared_ctx_lock(x)
|
||||
|
||||
/**
|
||||
* __cond_lock_shared() - function that conditionally acquires a context lock shared
|
||||
* @x: context lock instance pinter
|
||||
* @c: boolean expression
|
||||
*
|
||||
* Return: result of @c
|
||||
*
|
||||
* No-op function that conditionally acquires context lock instance @x with
|
||||
* shared access, if the boolean expression @c is true. The result of @c is the
|
||||
* return value.
|
||||
*/
|
||||
#define __cond_lock_shared(x, c) __try_acquire_shared_ctx_lock(x, c)
|
||||
|
||||
/**
|
||||
* __acquire_ret() - helper to acquire context lock of return value
|
||||
* @call: call expression
|
||||
|
|
|
|||
|
|
@ -49,9 +49,7 @@ static inline void lockref_init(struct lockref *lockref)
|
|||
void lockref_get(struct lockref *lockref);
|
||||
int lockref_put_return(struct lockref *lockref);
|
||||
bool lockref_get_not_zero(struct lockref *lockref);
|
||||
bool lockref_put_or_lock(struct lockref *lockref);
|
||||
#define lockref_put_or_lock(_lockref) \
|
||||
(!__cond_lock((_lockref)->lock, !lockref_put_or_lock(_lockref)))
|
||||
bool lockref_put_or_lock(struct lockref *lockref) __cond_acquires(false, &lockref->lock);
|
||||
|
||||
void lockref_mark_dead(struct lockref *lockref);
|
||||
bool lockref_get_not_dead(struct lockref *lockref);
|
||||
|
|
|
|||
|
|
@ -2975,15 +2975,8 @@ static inline pud_t pud_mkspecial(pud_t pud)
|
|||
}
|
||||
#endif /* CONFIG_ARCH_SUPPORTS_PUD_PFNMAP */
|
||||
|
||||
extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
|
||||
spinlock_t **ptl);
|
||||
static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
|
||||
spinlock_t **ptl)
|
||||
{
|
||||
pte_t *ptep;
|
||||
__cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl));
|
||||
return ptep;
|
||||
}
|
||||
extern pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
|
||||
spinlock_t **ptl);
|
||||
|
||||
#ifdef __PAGETABLE_P4D_FOLDED
|
||||
static inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
|
||||
|
|
@ -3337,31 +3330,15 @@ static inline bool pagetable_pte_ctor(struct mm_struct *mm,
|
|||
return true;
|
||||
}
|
||||
|
||||
pte_t *___pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp);
|
||||
static inline pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr,
|
||||
pmd_t *pmdvalp)
|
||||
{
|
||||
pte_t *pte;
|
||||
pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp);
|
||||
|
||||
__cond_lock(RCU, pte = ___pte_offset_map(pmd, addr, pmdvalp));
|
||||
return pte;
|
||||
}
|
||||
static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr)
|
||||
{
|
||||
return __pte_offset_map(pmd, addr, NULL);
|
||||
}
|
||||
|
||||
pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
|
||||
unsigned long addr, spinlock_t **ptlp);
|
||||
static inline pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
|
||||
unsigned long addr, spinlock_t **ptlp)
|
||||
{
|
||||
pte_t *pte;
|
||||
|
||||
__cond_lock(RCU, __cond_lock(*ptlp,
|
||||
pte = __pte_offset_map_lock(mm, pmd, addr, ptlp)));
|
||||
return pte;
|
||||
}
|
||||
pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
|
||||
unsigned long addr, spinlock_t **ptlp);
|
||||
|
||||
pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd,
|
||||
unsigned long addr, spinlock_t **ptlp);
|
||||
|
|
|
|||
|
|
@ -50,8 +50,8 @@ do { \
|
|||
* regardless of whether CONFIG_SMP or CONFIG_PREEMPT are set. The various
|
||||
* methods are defined as nops in the case they are not required.
|
||||
*/
|
||||
#define read_trylock(lock) __cond_lock_shared(lock, _raw_read_trylock(lock))
|
||||
#define write_trylock(lock) __cond_lock(lock, _raw_write_trylock(lock))
|
||||
#define read_trylock(lock) _raw_read_trylock(lock)
|
||||
#define write_trylock(lock) _raw_write_trylock(lock)
|
||||
|
||||
#define write_lock(lock) _raw_write_lock(lock)
|
||||
#define read_lock(lock) _raw_read_lock(lock)
|
||||
|
|
@ -113,12 +113,7 @@ do { \
|
|||
} while (0)
|
||||
#define write_unlock_bh(lock) _raw_write_unlock_bh(lock)
|
||||
|
||||
#define write_trylock_irqsave(lock, flags) \
|
||||
__cond_lock(lock, ({ \
|
||||
local_irq_save(flags); \
|
||||
_raw_write_trylock(lock) ? \
|
||||
1 : ({ local_irq_restore(flags); 0; }); \
|
||||
}))
|
||||
#define write_trylock_irqsave(lock, flags) _raw_write_trylock_irqsave(lock, &(flags))
|
||||
|
||||
#ifdef arch_rwlock_is_contended
|
||||
#define rwlock_is_contended(lock) \
|
||||
|
|
|
|||
|
|
@ -26,8 +26,8 @@ unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock)
|
|||
__acquires(lock);
|
||||
unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock)
|
||||
__acquires(lock);
|
||||
int __lockfunc _raw_read_trylock(rwlock_t *lock);
|
||||
int __lockfunc _raw_write_trylock(rwlock_t *lock);
|
||||
int __lockfunc _raw_read_trylock(rwlock_t *lock) __cond_acquires_shared(true, lock);
|
||||
int __lockfunc _raw_write_trylock(rwlock_t *lock) __cond_acquires(true, lock);
|
||||
void __lockfunc _raw_read_unlock(rwlock_t *lock) __releases_shared(lock);
|
||||
void __lockfunc _raw_write_unlock(rwlock_t *lock) __releases(lock);
|
||||
void __lockfunc _raw_read_unlock_bh(rwlock_t *lock) __releases_shared(lock);
|
||||
|
|
@ -41,6 +41,16 @@ void __lockfunc
|
|||
_raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
|
||||
__releases(lock);
|
||||
|
||||
static inline bool _raw_write_trylock_irqsave(rwlock_t *lock, unsigned long *flags)
|
||||
__cond_acquires(true, lock)
|
||||
{
|
||||
local_irq_save(*flags);
|
||||
if (_raw_write_trylock(lock))
|
||||
return true;
|
||||
local_irq_restore(*flags);
|
||||
return false;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_INLINE_READ_LOCK
|
||||
#define _raw_read_lock(lock) __raw_read_lock(lock)
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -26,11 +26,11 @@ do { \
|
|||
} while (0)
|
||||
|
||||
extern void rt_read_lock(rwlock_t *rwlock) __acquires_shared(rwlock);
|
||||
extern int rt_read_trylock(rwlock_t *rwlock);
|
||||
extern int rt_read_trylock(rwlock_t *rwlock) __cond_acquires_shared(true, rwlock);
|
||||
extern void rt_read_unlock(rwlock_t *rwlock) __releases_shared(rwlock);
|
||||
extern void rt_write_lock(rwlock_t *rwlock) __acquires(rwlock);
|
||||
extern void rt_write_lock_nested(rwlock_t *rwlock, int subclass) __acquires(rwlock);
|
||||
extern int rt_write_trylock(rwlock_t *rwlock);
|
||||
extern int rt_write_trylock(rwlock_t *rwlock) __cond_acquires(true, rwlock);
|
||||
extern void rt_write_unlock(rwlock_t *rwlock) __releases(rwlock);
|
||||
|
||||
static __always_inline void read_lock(rwlock_t *rwlock)
|
||||
|
|
@ -59,7 +59,7 @@ static __always_inline void read_lock_irq(rwlock_t *rwlock)
|
|||
flags = 0; \
|
||||
} while (0)
|
||||
|
||||
#define read_trylock(lock) __cond_lock_shared(lock, rt_read_trylock(lock))
|
||||
#define read_trylock(lock) rt_read_trylock(lock)
|
||||
|
||||
static __always_inline void read_unlock(rwlock_t *rwlock)
|
||||
__releases_shared(rwlock)
|
||||
|
|
@ -123,14 +123,15 @@ static __always_inline void write_lock_irq(rwlock_t *rwlock)
|
|||
flags = 0; \
|
||||
} while (0)
|
||||
|
||||
#define write_trylock(lock) __cond_lock(lock, rt_write_trylock(lock))
|
||||
#define write_trylock(lock) rt_write_trylock(lock)
|
||||
|
||||
#define write_trylock_irqsave(lock, flags) \
|
||||
__cond_lock(lock, ({ \
|
||||
typecheck(unsigned long, flags); \
|
||||
flags = 0; \
|
||||
rt_write_trylock(lock); \
|
||||
}))
|
||||
static __always_inline bool _write_trylock_irqsave(rwlock_t *rwlock, unsigned long *flags)
|
||||
__cond_acquires(true, rwlock)
|
||||
{
|
||||
*flags = 0;
|
||||
return rt_write_trylock(rwlock);
|
||||
}
|
||||
#define write_trylock_irqsave(lock, flags) _write_trylock_irqsave(lock, &(flags))
|
||||
|
||||
static __always_inline void write_unlock(rwlock_t *rwlock)
|
||||
__releases(rwlock)
|
||||
|
|
|
|||
|
|
@ -737,18 +737,8 @@ static inline int thread_group_empty(struct task_struct *p)
|
|||
#define delay_group_leader(p) \
|
||||
(thread_group_leader(p) && !thread_group_empty(p))
|
||||
|
||||
extern struct sighand_struct *__lock_task_sighand(struct task_struct *task,
|
||||
unsigned long *flags);
|
||||
|
||||
static inline struct sighand_struct *lock_task_sighand(struct task_struct *task,
|
||||
unsigned long *flags)
|
||||
{
|
||||
struct sighand_struct *ret;
|
||||
|
||||
ret = __lock_task_sighand(task, flags);
|
||||
(void)__cond_lock(&task->sighand->siglock, ret);
|
||||
return ret;
|
||||
}
|
||||
extern struct sighand_struct *lock_task_sighand(struct task_struct *task,
|
||||
unsigned long *flags);
|
||||
|
||||
static inline void unlock_task_sighand(struct task_struct *task,
|
||||
unsigned long *flags)
|
||||
|
|
|
|||
|
|
@ -213,7 +213,7 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
|
|||
* various methods are defined as nops in the case they are not
|
||||
* required.
|
||||
*/
|
||||
#define raw_spin_trylock(lock) __cond_lock(lock, _raw_spin_trylock(lock))
|
||||
#define raw_spin_trylock(lock) _raw_spin_trylock(lock)
|
||||
|
||||
#define raw_spin_lock(lock) _raw_spin_lock(lock)
|
||||
|
||||
|
|
@ -284,22 +284,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
|
|||
} while (0)
|
||||
#define raw_spin_unlock_bh(lock) _raw_spin_unlock_bh(lock)
|
||||
|
||||
#define raw_spin_trylock_bh(lock) \
|
||||
__cond_lock(lock, _raw_spin_trylock_bh(lock))
|
||||
#define raw_spin_trylock_bh(lock) _raw_spin_trylock_bh(lock)
|
||||
|
||||
#define raw_spin_trylock_irq(lock) \
|
||||
__cond_lock(lock, ({ \
|
||||
local_irq_disable(); \
|
||||
_raw_spin_trylock(lock) ? \
|
||||
1 : ({ local_irq_enable(); 0; }); \
|
||||
}))
|
||||
#define raw_spin_trylock_irq(lock) _raw_spin_trylock_irq(lock)
|
||||
|
||||
#define raw_spin_trylock_irqsave(lock, flags) \
|
||||
__cond_lock(lock, ({ \
|
||||
local_irq_save(flags); \
|
||||
_raw_spin_trylock(lock) ? \
|
||||
1 : ({ local_irq_restore(flags); 0; }); \
|
||||
}))
|
||||
#define raw_spin_trylock_irqsave(lock, flags) _raw_spin_trylock_irqsave(lock, &(flags))
|
||||
|
||||
#ifndef CONFIG_PREEMPT_RT
|
||||
/* Include rwlock functions for !RT */
|
||||
|
|
@ -433,8 +422,12 @@ static __always_inline int spin_trylock_irq(spinlock_t *lock)
|
|||
return raw_spin_trylock_irq(&lock->rlock);
|
||||
}
|
||||
|
||||
#define spin_trylock_irqsave(lock, flags) \
|
||||
__cond_lock(lock, raw_spin_trylock_irqsave(spinlock_check(lock), flags))
|
||||
static __always_inline bool _spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags)
|
||||
__cond_acquires(true, lock) __no_context_analysis
|
||||
{
|
||||
return raw_spin_trylock_irqsave(spinlock_check(lock), *flags);
|
||||
}
|
||||
#define spin_trylock_irqsave(lock, flags) _spin_trylock_irqsave(lock, &(flags))
|
||||
|
||||
/**
|
||||
* spin_is_locked() - Check whether a spinlock is locked.
|
||||
|
|
@ -512,23 +505,17 @@ static inline int rwlock_needbreak(rwlock_t *lock)
|
|||
* Decrements @atomic by 1. If the result is 0, returns true and locks
|
||||
* @lock. Returns false for all other cases.
|
||||
*/
|
||||
extern int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock);
|
||||
#define atomic_dec_and_lock(atomic, lock) \
|
||||
__cond_lock(lock, _atomic_dec_and_lock(atomic, lock))
|
||||
extern int atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock) __cond_acquires(true, lock);
|
||||
|
||||
extern int _atomic_dec_and_lock_irqsave(atomic_t *atomic, spinlock_t *lock,
|
||||
unsigned long *flags);
|
||||
#define atomic_dec_and_lock_irqsave(atomic, lock, flags) \
|
||||
__cond_lock(lock, _atomic_dec_and_lock_irqsave(atomic, lock, &(flags)))
|
||||
unsigned long *flags) __cond_acquires(true, lock);
|
||||
#define atomic_dec_and_lock_irqsave(atomic, lock, flags) _atomic_dec_and_lock_irqsave(atomic, lock, &(flags))
|
||||
|
||||
extern int _atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock);
|
||||
#define atomic_dec_and_raw_lock(atomic, lock) \
|
||||
__cond_lock(lock, _atomic_dec_and_raw_lock(atomic, lock))
|
||||
extern int atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock) __cond_acquires(true, lock);
|
||||
|
||||
extern int _atomic_dec_and_raw_lock_irqsave(atomic_t *atomic, raw_spinlock_t *lock,
|
||||
unsigned long *flags);
|
||||
#define atomic_dec_and_raw_lock_irqsave(atomic, lock, flags) \
|
||||
__cond_lock(lock, _atomic_dec_and_raw_lock_irqsave(atomic, lock, &(flags)))
|
||||
unsigned long *flags) __cond_acquires(true, lock);
|
||||
#define atomic_dec_and_raw_lock_irqsave(atomic, lock, flags) _atomic_dec_and_raw_lock_irqsave(atomic, lock, &(flags))
|
||||
|
||||
int __alloc_bucket_spinlocks(spinlock_t **locks, unsigned int *lock_mask,
|
||||
size_t max_size, unsigned int cpu_mult,
|
||||
|
|
|
|||
|
|
@ -95,6 +95,26 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static __always_inline bool _raw_spin_trylock_irq(raw_spinlock_t *lock)
|
||||
__cond_acquires(true, lock)
|
||||
{
|
||||
local_irq_disable();
|
||||
if (_raw_spin_trylock(lock))
|
||||
return true;
|
||||
local_irq_enable();
|
||||
return false;
|
||||
}
|
||||
|
||||
static __always_inline bool _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags)
|
||||
__cond_acquires(true, lock)
|
||||
{
|
||||
local_irq_save(*flags);
|
||||
if (_raw_spin_trylock(lock))
|
||||
return true;
|
||||
local_irq_restore(*flags);
|
||||
return false;
|
||||
}
|
||||
|
||||
/*
|
||||
* If lockdep is enabled then we use the non-preemption spin-ops
|
||||
* even on CONFIG_PREEMPTION, because lockdep assumes that interrupts are
|
||||
|
|
|
|||
|
|
@ -24,14 +24,11 @@
|
|||
* flags straight, to suppress compiler warnings of unused lock
|
||||
* variables, and to add the proper checker annotations:
|
||||
*/
|
||||
#define ___LOCK_void(lock) \
|
||||
do { (void)(lock); } while (0)
|
||||
|
||||
#define ___LOCK_(lock) \
|
||||
do { __acquire(lock); ___LOCK_void(lock); } while (0)
|
||||
do { __acquire(lock); (void)(lock); } while (0)
|
||||
|
||||
#define ___LOCK_shared(lock) \
|
||||
do { __acquire_shared(lock); ___LOCK_void(lock); } while (0)
|
||||
do { __acquire_shared(lock); (void)(lock); } while (0)
|
||||
|
||||
#define __LOCK(lock, ...) \
|
||||
do { preempt_disable(); ___LOCK_##__VA_ARGS__(lock); } while (0)
|
||||
|
|
@ -78,10 +75,56 @@
|
|||
#define _raw_spin_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags)
|
||||
#define _raw_read_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags, shared)
|
||||
#define _raw_write_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags)
|
||||
#define _raw_spin_trylock(lock) ({ __LOCK(lock, void); 1; })
|
||||
#define _raw_read_trylock(lock) ({ __LOCK(lock, void); 1; })
|
||||
#define _raw_write_trylock(lock) ({ __LOCK(lock, void); 1; })
|
||||
#define _raw_spin_trylock_bh(lock) ({ __LOCK_BH(lock, void); 1; })
|
||||
|
||||
static __always_inline int _raw_spin_trylock(raw_spinlock_t *lock)
|
||||
__cond_acquires(true, lock)
|
||||
{
|
||||
__LOCK(lock);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static __always_inline int _raw_spin_trylock_bh(raw_spinlock_t *lock)
|
||||
__cond_acquires(true, lock)
|
||||
{
|
||||
__LOCK_BH(lock);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static __always_inline int _raw_spin_trylock_irq(raw_spinlock_t *lock)
|
||||
__cond_acquires(true, lock)
|
||||
{
|
||||
__LOCK_IRQ(lock);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static __always_inline int _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags)
|
||||
__cond_acquires(true, lock)
|
||||
{
|
||||
__LOCK_IRQSAVE(lock, *(flags));
|
||||
return 1;
|
||||
}
|
||||
|
||||
static __always_inline int _raw_read_trylock(rwlock_t *lock)
|
||||
__cond_acquires_shared(true, lock)
|
||||
{
|
||||
__LOCK(lock, shared);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static __always_inline int _raw_write_trylock(rwlock_t *lock)
|
||||
__cond_acquires(true, lock)
|
||||
{
|
||||
__LOCK(lock);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static __always_inline int _raw_write_trylock_irqsave(rwlock_t *lock, unsigned long *flags)
|
||||
__cond_acquires(true, lock)
|
||||
{
|
||||
__LOCK_IRQSAVE(lock, *(flags));
|
||||
return 1;
|
||||
}
|
||||
|
||||
#define _raw_spin_unlock(lock) __UNLOCK(lock)
|
||||
#define _raw_read_unlock(lock) __UNLOCK(lock, shared)
|
||||
#define _raw_write_unlock(lock) __UNLOCK(lock)
|
||||
|
|
|
|||
|
|
@ -37,8 +37,8 @@ extern void rt_spin_lock_nested(spinlock_t *lock, int subclass) __acquires(lock)
|
|||
extern void rt_spin_lock_nest_lock(spinlock_t *lock, struct lockdep_map *nest_lock) __acquires(lock);
|
||||
extern void rt_spin_unlock(spinlock_t *lock) __releases(lock);
|
||||
extern void rt_spin_lock_unlock(spinlock_t *lock);
|
||||
extern int rt_spin_trylock_bh(spinlock_t *lock);
|
||||
extern int rt_spin_trylock(spinlock_t *lock);
|
||||
extern int rt_spin_trylock_bh(spinlock_t *lock) __cond_acquires(true, lock);
|
||||
extern int rt_spin_trylock(spinlock_t *lock) __cond_acquires(true, lock);
|
||||
|
||||
static __always_inline void spin_lock(spinlock_t *lock)
|
||||
__acquires(lock)
|
||||
|
|
@ -130,21 +130,19 @@ static __always_inline void spin_unlock_irqrestore(spinlock_t *lock,
|
|||
rt_spin_unlock(lock);
|
||||
}
|
||||
|
||||
#define spin_trylock(lock) \
|
||||
__cond_lock(lock, rt_spin_trylock(lock))
|
||||
#define spin_trylock(lock) rt_spin_trylock(lock)
|
||||
|
||||
#define spin_trylock_bh(lock) \
|
||||
__cond_lock(lock, rt_spin_trylock_bh(lock))
|
||||
#define spin_trylock_bh(lock) rt_spin_trylock_bh(lock)
|
||||
|
||||
#define spin_trylock_irq(lock) \
|
||||
__cond_lock(lock, rt_spin_trylock(lock))
|
||||
#define spin_trylock_irq(lock) rt_spin_trylock(lock)
|
||||
|
||||
#define spin_trylock_irqsave(lock, flags) \
|
||||
__cond_lock(lock, ({ \
|
||||
typecheck(unsigned long, flags); \
|
||||
flags = 0; \
|
||||
rt_spin_trylock(lock); \
|
||||
}))
|
||||
static __always_inline bool _spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags)
|
||||
__cond_acquires(true, lock)
|
||||
{
|
||||
*flags = 0;
|
||||
return rt_spin_trylock(lock);
|
||||
}
|
||||
#define spin_trylock_irqsave(lock, flags) _spin_trylock_irqsave(lock, &(flags))
|
||||
|
||||
#define spin_is_contended(lock) (((void)(lock), 0))
|
||||
|
||||
|
|
|
|||
|
|
@ -1355,8 +1355,8 @@ int zap_other_threads(struct task_struct *p)
|
|||
return count;
|
||||
}
|
||||
|
||||
struct sighand_struct *__lock_task_sighand(struct task_struct *tsk,
|
||||
unsigned long *flags)
|
||||
struct sighand_struct *lock_task_sighand(struct task_struct *tsk,
|
||||
unsigned long *flags)
|
||||
{
|
||||
struct sighand_struct *sighand;
|
||||
|
||||
|
|
|
|||
|
|
@ -66,14 +66,7 @@ static const struct k_clock clock_realtime, clock_monotonic;
|
|||
#error "SIGEV_THREAD_ID must not share bit with other SIGEV values!"
|
||||
#endif
|
||||
|
||||
static struct k_itimer *__lock_timer(timer_t timer_id);
|
||||
|
||||
#define lock_timer(tid) \
|
||||
({ struct k_itimer *__timr; \
|
||||
__cond_lock(&__timr->it_lock, __timr = __lock_timer(tid)); \
|
||||
__timr; \
|
||||
})
|
||||
|
||||
static struct k_itimer *lock_timer(timer_t timer_id);
|
||||
static inline void unlock_timer(struct k_itimer *timr)
|
||||
{
|
||||
if (likely((timr)))
|
||||
|
|
@ -85,7 +78,7 @@ static inline void unlock_timer(struct k_itimer *timr)
|
|||
|
||||
#define scoped_timer (scope)
|
||||
|
||||
DEFINE_CLASS(lock_timer, struct k_itimer *, unlock_timer(_T), __lock_timer(id), timer_t id);
|
||||
DEFINE_CLASS(lock_timer, struct k_itimer *, unlock_timer(_T), lock_timer(id), timer_t id);
|
||||
DEFINE_CLASS_IS_COND_GUARD(lock_timer);
|
||||
|
||||
static struct timer_hash_bucket *hash_bucket(struct signal_struct *sig, unsigned int nr)
|
||||
|
|
@ -600,7 +593,7 @@ COMPAT_SYSCALL_DEFINE3(timer_create, clockid_t, which_clock,
|
|||
}
|
||||
#endif
|
||||
|
||||
static struct k_itimer *__lock_timer(timer_t timer_id)
|
||||
static struct k_itimer *lock_timer(timer_t timer_id)
|
||||
{
|
||||
struct k_itimer *timr;
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
* because the spin-lock and the decrement must be
|
||||
* "atomic".
|
||||
*/
|
||||
int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
|
||||
int atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
|
||||
{
|
||||
/* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
|
||||
if (atomic_add_unless(atomic, -1, 1))
|
||||
|
|
@ -32,7 +32,7 @@ int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
|
|||
return 0;
|
||||
}
|
||||
|
||||
EXPORT_SYMBOL(_atomic_dec_and_lock);
|
||||
EXPORT_SYMBOL(atomic_dec_and_lock);
|
||||
|
||||
int _atomic_dec_and_lock_irqsave(atomic_t *atomic, spinlock_t *lock,
|
||||
unsigned long *flags)
|
||||
|
|
@ -50,7 +50,7 @@ int _atomic_dec_and_lock_irqsave(atomic_t *atomic, spinlock_t *lock,
|
|||
}
|
||||
EXPORT_SYMBOL(_atomic_dec_and_lock_irqsave);
|
||||
|
||||
int _atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock)
|
||||
int atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock)
|
||||
{
|
||||
/* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
|
||||
if (atomic_add_unless(atomic, -1, 1))
|
||||
|
|
@ -63,7 +63,7 @@ int _atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock)
|
|||
raw_spin_unlock(lock);
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL(_atomic_dec_and_raw_lock);
|
||||
EXPORT_SYMBOL(atomic_dec_and_raw_lock);
|
||||
|
||||
int _atomic_dec_and_raw_lock_irqsave(atomic_t *atomic, raw_spinlock_t *lock,
|
||||
unsigned long *flags)
|
||||
|
|
|
|||
|
|
@ -105,7 +105,6 @@ EXPORT_SYMBOL(lockref_put_return);
|
|||
* @lockref: pointer to lockref structure
|
||||
* Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
|
||||
*/
|
||||
#undef lockref_put_or_lock
|
||||
bool lockref_put_or_lock(struct lockref *lockref)
|
||||
{
|
||||
CMPXCHG_LOOP(
|
||||
|
|
|
|||
|
|
@ -2210,8 +2210,8 @@ static pmd_t *walk_to_pmd(struct mm_struct *mm, unsigned long addr)
|
|||
return pmd;
|
||||
}
|
||||
|
||||
pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
|
||||
spinlock_t **ptl)
|
||||
pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
|
||||
spinlock_t **ptl)
|
||||
{
|
||||
pmd_t *pmd = walk_to_pmd(mm, addr);
|
||||
|
||||
|
|
|
|||
|
|
@ -280,7 +280,7 @@ static unsigned long pmdp_get_lockless_start(void) { return 0; }
|
|||
static void pmdp_get_lockless_end(unsigned long irqflags) { }
|
||||
#endif
|
||||
|
||||
pte_t *___pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp)
|
||||
pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp)
|
||||
{
|
||||
unsigned long irqflags;
|
||||
pmd_t pmdval;
|
||||
|
|
@ -332,13 +332,12 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd,
|
|||
}
|
||||
|
||||
/*
|
||||
* pte_offset_map_lock(mm, pmd, addr, ptlp), and its internal implementation
|
||||
* __pte_offset_map_lock() below, is usually called with the pmd pointer for
|
||||
* addr, reached by walking down the mm's pgd, p4d, pud for addr: either while
|
||||
* holding mmap_lock or vma lock for read or for write; or in truncate or rmap
|
||||
* context, while holding file's i_mmap_lock or anon_vma lock for read (or for
|
||||
* write). In a few cases, it may be used with pmd pointing to a pmd_t already
|
||||
* copied to or constructed on the stack.
|
||||
* pte_offset_map_lock(mm, pmd, addr, ptlp) is usually called with the pmd
|
||||
* pointer for addr, reached by walking down the mm's pgd, p4d, pud for addr:
|
||||
* either while holding mmap_lock or vma lock for read or for write; or in
|
||||
* truncate or rmap context, while holding file's i_mmap_lock or anon_vma lock
|
||||
* for read (or for write). In a few cases, it may be used with pmd pointing to
|
||||
* a pmd_t already copied to or constructed on the stack.
|
||||
*
|
||||
* When successful, it returns the pte pointer for addr, with its page table
|
||||
* kmapped if necessary (when CONFIG_HIGHPTE), and locked against concurrent
|
||||
|
|
@ -389,8 +388,8 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd,
|
|||
* table, and may not use RCU at all: "outsiders" like khugepaged should avoid
|
||||
* pte_offset_map() and co once the vma is detached from mm or mm_users is zero.
|
||||
*/
|
||||
pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
|
||||
unsigned long addr, spinlock_t **ptlp)
|
||||
pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
|
||||
unsigned long addr, spinlock_t **ptlp)
|
||||
{
|
||||
spinlock_t *ptl;
|
||||
pmd_t pmdval;
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@
|
|||
# define __releases(x) __attribute__((context(x,1,0)))
|
||||
# define __acquire(x) __context__(x,1)
|
||||
# define __release(x) __context__(x,-1)
|
||||
# define __cond_lock(x,c) ((c) ? ({ __acquire(x); 1; }) : 0)
|
||||
#else /* __CHECKER__ */
|
||||
/* context/locking */
|
||||
# define __must_hold(x)
|
||||
|
|
@ -28,7 +27,6 @@
|
|||
# define __releases(x)
|
||||
# define __acquire(x) (void)0
|
||||
# define __release(x) (void)0
|
||||
# define __cond_lock(x,c) (c)
|
||||
#endif /* __CHECKER__ */
|
||||
|
||||
/* Compiler specific macros. */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue