From f7d71f2052555ae57b47322f2c2f6c29ff2438ae Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Fri, 19 Jun 2015 11:50:00 -0400 Subject: locking/qrwlock: Rename functions to queued_*() To sync up with the naming convention used in qspinlock, all the qrwlock functions were renamed to started with "queued" instead of "queue". Signed-off-by: Waiman Long Signed-off-by: Peter Zijlstra (Intel) Cc: Arnd Bergmann Cc: Douglas Hatch Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Scott J Norton Cc: Thomas Gleixner Cc: Will Deacon Link: http://lkml.kernel.org/r/1434729002-57724-2-git-send-email-Waiman.Long@hp.com Signed-off-by: Ingo Molnar --- arch/x86/include/asm/qrwlock.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/qrwlock.h b/arch/x86/include/asm/qrwlock.h index ae0e241e228b..a8810bf135ab 100644 --- a/arch/x86/include/asm/qrwlock.h +++ b/arch/x86/include/asm/qrwlock.h @@ -4,8 +4,8 @@ #include #ifndef CONFIG_X86_PPRO_FENCE -#define queue_write_unlock queue_write_unlock -static inline void queue_write_unlock(struct qrwlock *lock) +#define queued_write_unlock queued_write_unlock +static inline void queued_write_unlock(struct qrwlock *lock) { barrier(); ACCESS_ONCE(*(u8 *)&lock->cnts) = 0; -- cgit v1.2.3 From 1b0b7c1762679a2f8bc359da95649249dfcf4195 Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Wed, 1 Jul 2015 13:29:48 -0700 Subject: rtmutex: Delete scriptable tester No one uses this anymore, and this is not the first time the idea of replacing it with a (now possible) userspace side. Lock stealing logic was removed long ago in when the lock was granted to the highest prio. Signed-off-by: Davidlohr Bueso Cc: Darren Hart Cc: Steven Rostedt Cc: Mike Galbraith Cc: Paul E. McKenney Cc: Sebastian Andrzej Siewior Cc: Davidlohr Bueso Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1435782588-4177-2-git-send-email-dave@stgolabs.net Signed-off-by: Thomas Gleixner --- arch/parisc/configs/c8000_defconfig | 1 - arch/parisc/configs/generic-32bit_defconfig | 1 - arch/xtensa/configs/iss_defconfig | 1 - kernel/locking/Makefile | 1 - kernel/locking/rtmutex-tester.c | 420 --------------------- kernel/locking/rtmutex.c | 2 +- kernel/locking/rtmutex_common.h | 22 -- lib/Kconfig.debug | 6 - scripts/rt-tester/check-all.sh | 21 -- scripts/rt-tester/rt-tester.py | 218 ----------- scripts/rt-tester/t2-l1-2rt-sameprio.tst | 94 ----- scripts/rt-tester/t2-l1-pi.tst | 77 ---- scripts/rt-tester/t2-l1-signal.tst | 72 ---- scripts/rt-tester/t2-l2-2rt-deadlock.tst | 84 ----- scripts/rt-tester/t3-l1-pi-1rt.tst | 87 ----- scripts/rt-tester/t3-l1-pi-2rt.tst | 88 ----- scripts/rt-tester/t3-l1-pi-3rt.tst | 87 ----- scripts/rt-tester/t3-l1-pi-signal.tst | 93 ----- scripts/rt-tester/t3-l1-pi-steal.tst | 91 ----- scripts/rt-tester/t3-l2-pi.tst | 87 ----- scripts/rt-tester/t4-l2-pi-deboost.tst | 118 ------ .../rt-tester/t5-l4-pi-boost-deboost-setsched.tst | 178 --------- scripts/rt-tester/t5-l4-pi-boost-deboost.tst | 138 ------- 23 files changed, 1 insertion(+), 1986 deletions(-) delete mode 100644 kernel/locking/rtmutex-tester.c delete mode 100644 scripts/rt-tester/check-all.sh delete mode 100755 scripts/rt-tester/rt-tester.py delete mode 100644 scripts/rt-tester/t2-l1-2rt-sameprio.tst delete mode 100644 scripts/rt-tester/t2-l1-pi.tst delete mode 100644 scripts/rt-tester/t2-l1-signal.tst delete mode 100644 scripts/rt-tester/t2-l2-2rt-deadlock.tst delete mode 100644 scripts/rt-tester/t3-l1-pi-1rt.tst delete mode 100644 scripts/rt-tester/t3-l1-pi-2rt.tst delete mode 100644 scripts/rt-tester/t3-l1-pi-3rt.tst delete mode 100644 scripts/rt-tester/t3-l1-pi-signal.tst delete mode 100644 scripts/rt-tester/t3-l1-pi-steal.tst delete mode 100644 scripts/rt-tester/t3-l2-pi.tst delete mode 100644 scripts/rt-tester/t4-l2-pi-deboost.tst delete mode 100644 scripts/rt-tester/t5-l4-pi-boost-deboost-setsched.tst delete mode 100644 scripts/rt-tester/t5-l4-pi-boost-deboost.tst (limited to 'arch') diff --git a/arch/parisc/configs/c8000_defconfig b/arch/parisc/configs/c8000_defconfig index 269c23d23fcb..1a8f6f95689e 100644 --- a/arch/parisc/configs/c8000_defconfig +++ b/arch/parisc/configs/c8000_defconfig @@ -242,7 +242,6 @@ CONFIG_LOCKUP_DETECTOR=y CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y CONFIG_PANIC_ON_OOPS=y CONFIG_DEBUG_RT_MUTEXES=y -CONFIG_RT_MUTEX_TESTER=y CONFIG_PROVE_RCU_DELAY=y CONFIG_DEBUG_BLOCK_EXT_DEVT=y CONFIG_LATENCYTOP=y diff --git a/arch/parisc/configs/generic-32bit_defconfig b/arch/parisc/configs/generic-32bit_defconfig index 33b148f825ba..0ffb08ff5125 100644 --- a/arch/parisc/configs/generic-32bit_defconfig +++ b/arch/parisc/configs/generic-32bit_defconfig @@ -295,7 +295,6 @@ CONFIG_DEBUG_SHIRQ=y CONFIG_DETECT_HUNG_TASK=y CONFIG_TIMER_STATS=y CONFIG_DEBUG_RT_MUTEXES=y -CONFIG_RT_MUTEX_TESTER=y CONFIG_DEBUG_SPINLOCK=y CONFIG_DEBUG_MUTEXES=y CONFIG_RCU_CPU_STALL_INFO=y diff --git a/arch/xtensa/configs/iss_defconfig b/arch/xtensa/configs/iss_defconfig index e4d193e7a300..f3dfe0d921c2 100644 --- a/arch/xtensa/configs/iss_defconfig +++ b/arch/xtensa/configs/iss_defconfig @@ -616,7 +616,6 @@ CONFIG_SCHED_DEBUG=y # CONFIG_SLUB_DEBUG_ON is not set # CONFIG_SLUB_STATS is not set # CONFIG_DEBUG_RT_MUTEXES is not set -# CONFIG_RT_MUTEX_TESTER is not set # CONFIG_DEBUG_SPINLOCK is not set # CONFIG_DEBUG_MUTEXES is not set # CONFIG_DEBUG_SPINLOCK_SLEEP is not set diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile index 7dd5c9918e4c..36942047ffc0 100644 --- a/kernel/locking/Makefile +++ b/kernel/locking/Makefile @@ -20,7 +20,6 @@ obj-$(CONFIG_PROVE_LOCKING) += spinlock.o obj-$(CONFIG_QUEUED_SPINLOCKS) += qspinlock.o obj-$(CONFIG_RT_MUTEXES) += rtmutex.o obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o -obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o obj-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o diff --git a/kernel/locking/rtmutex-tester.c b/kernel/locking/rtmutex-tester.c deleted file mode 100644 index 1d96dd0d93c1..000000000000 --- a/kernel/locking/rtmutex-tester.c +++ /dev/null @@ -1,420 +0,0 @@ -/* - * RT-Mutex-tester: scriptable tester for rt mutexes - * - * started by Thomas Gleixner: - * - * Copyright (C) 2006, Timesys Corp., Thomas Gleixner - * - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "rtmutex.h" - -#define MAX_RT_TEST_THREADS 8 -#define MAX_RT_TEST_MUTEXES 8 - -static spinlock_t rttest_lock; -static atomic_t rttest_event; - -struct test_thread_data { - int opcode; - int opdata; - int mutexes[MAX_RT_TEST_MUTEXES]; - int event; - struct device dev; -}; - -static struct test_thread_data thread_data[MAX_RT_TEST_THREADS]; -static struct task_struct *threads[MAX_RT_TEST_THREADS]; -static struct rt_mutex mutexes[MAX_RT_TEST_MUTEXES]; - -enum test_opcodes { - RTTEST_NOP = 0, - RTTEST_SCHEDOT, /* 1 Sched other, data = nice */ - RTTEST_SCHEDRT, /* 2 Sched fifo, data = prio */ - RTTEST_LOCK, /* 3 Lock uninterruptible, data = lockindex */ - RTTEST_LOCKNOWAIT, /* 4 Lock uninterruptible no wait in wakeup, data = lockindex */ - RTTEST_LOCKINT, /* 5 Lock interruptible, data = lockindex */ - RTTEST_LOCKINTNOWAIT, /* 6 Lock interruptible no wait in wakeup, data = lockindex */ - RTTEST_LOCKCONT, /* 7 Continue locking after the wakeup delay */ - RTTEST_UNLOCK, /* 8 Unlock, data = lockindex */ - /* 9, 10 - reserved for BKL commemoration */ - RTTEST_SIGNAL = 11, /* 11 Signal other test thread, data = thread id */ - RTTEST_RESETEVENT = 98, /* 98 Reset event counter */ - RTTEST_RESET = 99, /* 99 Reset all pending operations */ -}; - -static int handle_op(struct test_thread_data *td, int lockwakeup) -{ - int i, id, ret = -EINVAL; - - switch(td->opcode) { - - case RTTEST_NOP: - return 0; - - case RTTEST_LOCKCONT: - td->mutexes[td->opdata] = 1; - td->event = atomic_add_return(1, &rttest_event); - return 0; - - case RTTEST_RESET: - for (i = 0; i < MAX_RT_TEST_MUTEXES; i++) { - if (td->mutexes[i] == 4) { - rt_mutex_unlock(&mutexes[i]); - td->mutexes[i] = 0; - } - } - return 0; - - case RTTEST_RESETEVENT: - atomic_set(&rttest_event, 0); - return 0; - - default: - if (lockwakeup) - return ret; - } - - switch(td->opcode) { - - case RTTEST_LOCK: - case RTTEST_LOCKNOWAIT: - id = td->opdata; - if (id < 0 || id >= MAX_RT_TEST_MUTEXES) - return ret; - - td->mutexes[id] = 1; - td->event = atomic_add_return(1, &rttest_event); - rt_mutex_lock(&mutexes[id]); - td->event = atomic_add_return(1, &rttest_event); - td->mutexes[id] = 4; - return 0; - - case RTTEST_LOCKINT: - case RTTEST_LOCKINTNOWAIT: - id = td->opdata; - if (id < 0 || id >= MAX_RT_TEST_MUTEXES) - return ret; - - td->mutexes[id] = 1; - td->event = atomic_add_return(1, &rttest_event); - ret = rt_mutex_lock_interruptible(&mutexes[id], 0); - td->event = atomic_add_return(1, &rttest_event); - td->mutexes[id] = ret ? 0 : 4; - return ret ? -EINTR : 0; - - case RTTEST_UNLOCK: - id = td->opdata; - if (id < 0 || id >= MAX_RT_TEST_MUTEXES || td->mutexes[id] != 4) - return ret; - - td->event = atomic_add_return(1, &rttest_event); - rt_mutex_unlock(&mutexes[id]); - td->event = atomic_add_return(1, &rttest_event); - td->mutexes[id] = 0; - return 0; - - default: - break; - } - return ret; -} - -/* - * Schedule replacement for rtsem_down(). Only called for threads with - * PF_MUTEX_TESTER set. - * - * This allows us to have finegrained control over the event flow. - * - */ -void schedule_rt_mutex_test(struct rt_mutex *mutex) -{ - int tid, op, dat; - struct test_thread_data *td; - - /* We have to lookup the task */ - for (tid = 0; tid < MAX_RT_TEST_THREADS; tid++) { - if (threads[tid] == current) - break; - } - - BUG_ON(tid == MAX_RT_TEST_THREADS); - - td = &thread_data[tid]; - - op = td->opcode; - dat = td->opdata; - - switch (op) { - case RTTEST_LOCK: - case RTTEST_LOCKINT: - case RTTEST_LOCKNOWAIT: - case RTTEST_LOCKINTNOWAIT: - if (mutex != &mutexes[dat]) - break; - - if (td->mutexes[dat] != 1) - break; - - td->mutexes[dat] = 2; - td->event = atomic_add_return(1, &rttest_event); - break; - - default: - break; - } - - schedule(); - - - switch (op) { - case RTTEST_LOCK: - case RTTEST_LOCKINT: - if (mutex != &mutexes[dat]) - return; - - if (td->mutexes[dat] != 2) - return; - - td->mutexes[dat] = 3; - td->event = atomic_add_return(1, &rttest_event); - break; - - case RTTEST_LOCKNOWAIT: - case RTTEST_LOCKINTNOWAIT: - if (mutex != &mutexes[dat]) - return; - - if (td->mutexes[dat] != 2) - return; - - td->mutexes[dat] = 1; - td->event = atomic_add_return(1, &rttest_event); - return; - - default: - return; - } - - td->opcode = 0; - - for (;;) { - set_current_state(TASK_INTERRUPTIBLE); - - if (td->opcode > 0) { - int ret; - - set_current_state(TASK_RUNNING); - ret = handle_op(td, 1); - set_current_state(TASK_INTERRUPTIBLE); - if (td->opcode == RTTEST_LOCKCONT) - break; - td->opcode = ret; - } - - /* Wait for the next command to be executed */ - schedule(); - } - - /* Restore previous command and data */ - td->opcode = op; - td->opdata = dat; -} - -static int test_func(void *data) -{ - struct test_thread_data *td = data; - int ret; - - current->flags |= PF_MUTEX_TESTER; - set_freezable(); - allow_signal(SIGHUP); - - for(;;) { - - set_current_state(TASK_INTERRUPTIBLE); - - if (td->opcode > 0) { - set_current_state(TASK_RUNNING); - ret = handle_op(td, 0); - set_current_state(TASK_INTERRUPTIBLE); - td->opcode = ret; - } - - /* Wait for the next command to be executed */ - schedule(); - try_to_freeze(); - - if (signal_pending(current)) - flush_signals(current); - - if(kthread_should_stop()) - break; - } - return 0; -} - -/** - * sysfs_test_command - interface for test commands - * @dev: thread reference - * @buf: command for actual step - * @count: length of buffer - * - * command syntax: - * - * opcode:data - */ -static ssize_t sysfs_test_command(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct sched_param schedpar; - struct test_thread_data *td; - char cmdbuf[32]; - int op, dat, tid, ret; - - td = container_of(dev, struct test_thread_data, dev); - tid = td->dev.id; - - /* strings from sysfs write are not 0 terminated! */ - if (count >= sizeof(cmdbuf)) - return -EINVAL; - - /* strip of \n: */ - if (buf[count-1] == '\n') - count--; - if (count < 1) - return -EINVAL; - - memcpy(cmdbuf, buf, count); - cmdbuf[count] = 0; - - if (sscanf(cmdbuf, "%d:%d", &op, &dat) != 2) - return -EINVAL; - - switch (op) { - case RTTEST_SCHEDOT: - schedpar.sched_priority = 0; - ret = sched_setscheduler(threads[tid], SCHED_NORMAL, &schedpar); - if (ret) - return ret; - set_user_nice(current, 0); - break; - - case RTTEST_SCHEDRT: - schedpar.sched_priority = dat; - ret = sched_setscheduler(threads[tid], SCHED_FIFO, &schedpar); - if (ret) - return ret; - break; - - case RTTEST_SIGNAL: - send_sig(SIGHUP, threads[tid], 0); - break; - - default: - if (td->opcode > 0) - return -EBUSY; - td->opdata = dat; - td->opcode = op; - wake_up_process(threads[tid]); - } - - return count; -} - -/** - * sysfs_test_status - sysfs interface for rt tester - * @dev: thread to query - * @buf: char buffer to be filled with thread status info - */ -static ssize_t sysfs_test_status(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct test_thread_data *td; - struct task_struct *tsk; - char *curr = buf; - int i; - - td = container_of(dev, struct test_thread_data, dev); - tsk = threads[td->dev.id]; - - spin_lock(&rttest_lock); - - curr += sprintf(curr, - "O: %4d, E:%8d, S: 0x%08lx, P: %4d, N: %4d, B: %p, M:", - td->opcode, td->event, tsk->state, - (MAX_RT_PRIO - 1) - tsk->prio, - (MAX_RT_PRIO - 1) - tsk->normal_prio, - tsk->pi_blocked_on); - - for (i = MAX_RT_TEST_MUTEXES - 1; i >=0 ; i--) - curr += sprintf(curr, "%d", td->mutexes[i]); - - spin_unlock(&rttest_lock); - - curr += sprintf(curr, ", T: %p, R: %p\n", tsk, - mutexes[td->dev.id].owner); - - return curr - buf; -} - -static DEVICE_ATTR(status, S_IRUSR, sysfs_test_status, NULL); -static DEVICE_ATTR(command, S_IWUSR, NULL, sysfs_test_command); - -static struct bus_type rttest_subsys = { - .name = "rttest", - .dev_name = "rttest", -}; - -static int init_test_thread(int id) -{ - thread_data[id].dev.bus = &rttest_subsys; - thread_data[id].dev.id = id; - - threads[id] = kthread_run(test_func, &thread_data[id], "rt-test-%d", id); - if (IS_ERR(threads[id])) - return PTR_ERR(threads[id]); - - return device_register(&thread_data[id].dev); -} - -static int init_rttest(void) -{ - int ret, i; - - spin_lock_init(&rttest_lock); - - for (i = 0; i < MAX_RT_TEST_MUTEXES; i++) - rt_mutex_init(&mutexes[i]); - - ret = subsys_system_register(&rttest_subsys, NULL); - if (ret) - return ret; - - for (i = 0; i < MAX_RT_TEST_THREADS; i++) { - ret = init_test_thread(i); - if (ret) - break; - ret = device_create_file(&thread_data[i].dev, &dev_attr_status); - if (ret) - break; - ret = device_create_file(&thread_data[i].dev, &dev_attr_command); - if (ret) - break; - } - - printk("Initializing RT-Tester: %s\n", ret ? "Failed" : "OK" ); - - return ret; -} - -device_initcall(init_rttest); diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c index 5674b073473c..7781d801212f 100644 --- a/kernel/locking/rtmutex.c +++ b/kernel/locking/rtmutex.c @@ -1120,7 +1120,7 @@ __rt_mutex_slowlock(struct rt_mutex *lock, int state, debug_rt_mutex_print_deadlock(waiter); - schedule_rt_mutex(lock); + schedule(); raw_spin_lock(&lock->wait_lock); set_current_state(state); diff --git a/kernel/locking/rtmutex_common.h b/kernel/locking/rtmutex_common.h index 7844f8f0e639..4f5f83c7d2d3 100644 --- a/kernel/locking/rtmutex_common.h +++ b/kernel/locking/rtmutex_common.h @@ -14,28 +14,6 @@ #include -/* - * The rtmutex in kernel tester is independent of rtmutex debugging. We - * call schedule_rt_mutex_test() instead of schedule() for the tasks which - * belong to the tester. That way we can delay the wakeup path of those - * threads to provoke lock stealing and testing of complex boosting scenarios. - */ -#ifdef CONFIG_RT_MUTEX_TESTER - -extern void schedule_rt_mutex_test(struct rt_mutex *lock); - -#define schedule_rt_mutex(_lock) \ - do { \ - if (!(current->flags & PF_MUTEX_TESTER)) \ - schedule(); \ - else \ - schedule_rt_mutex_test(_lock); \ - } while (0) - -#else -# define schedule_rt_mutex(_lock) schedule() -#endif - /* * This is the control structure for tasks blocked on a rt_mutex, * which is allocated on the kernel stack on of the blocked task. diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 22554d6f720f..e7b5b654ed8d 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -916,12 +916,6 @@ config DEBUG_RT_MUTEXES This allows rt mutex semantics violations and rt mutex related deadlocks (lockups) to be detected and reported automatically. -config RT_MUTEX_TESTER - bool "Built-in scriptable tester for rt-mutexes" - depends on DEBUG_KERNEL && RT_MUTEXES && BROKEN - help - This option enables a rt-mutex tester. - config DEBUG_SPINLOCK bool "Spinlock and rw-lock debugging: basic checks" depends on DEBUG_KERNEL diff --git a/scripts/rt-tester/check-all.sh b/scripts/rt-tester/check-all.sh deleted file mode 100644 index 6b5c83baf148..000000000000 --- a/scripts/rt-tester/check-all.sh +++ /dev/null @@ -1,21 +0,0 @@ - - -function testit () -{ - printf "%-30s: " $1 - ./rt-tester.py $1 | grep Pass -} - -testit t2-l1-2rt-sameprio.tst -testit t2-l1-pi.tst -testit t2-l1-signal.tst -#testit t2-l2-2rt-deadlock.tst -testit t3-l1-pi-1rt.tst -testit t3-l1-pi-2rt.tst -testit t3-l1-pi-3rt.tst -testit t3-l1-pi-signal.tst -testit t3-l1-pi-steal.tst -testit t3-l2-pi.tst -testit t4-l2-pi-deboost.tst -testit t5-l4-pi-boost-deboost.tst -testit t5-l4-pi-boost-deboost-setsched.tst diff --git a/scripts/rt-tester/rt-tester.py b/scripts/rt-tester/rt-tester.py deleted file mode 100755 index 6d916c2a45a5..000000000000 --- a/scripts/rt-tester/rt-tester.py +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/python -# -# rt-mutex tester -# -# (C) 2006 Thomas Gleixner -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -import os -import sys -import getopt -import shutil -import string - -# Globals -quiet = 0 -test = 0 -comments = 0 - -sysfsprefix = "/sys/devices/system/rttest/rttest" -statusfile = "/status" -commandfile = "/command" - -# Command opcodes -cmd_opcodes = { - "schedother" : "1", - "schedfifo" : "2", - "lock" : "3", - "locknowait" : "4", - "lockint" : "5", - "lockintnowait" : "6", - "lockcont" : "7", - "unlock" : "8", - "signal" : "11", - "resetevent" : "98", - "reset" : "99", - } - -test_opcodes = { - "prioeq" : ["P" , "eq" , None], - "priolt" : ["P" , "lt" , None], - "priogt" : ["P" , "gt" , None], - "nprioeq" : ["N" , "eq" , None], - "npriolt" : ["N" , "lt" , None], - "npriogt" : ["N" , "gt" , None], - "unlocked" : ["M" , "eq" , 0], - "trylock" : ["M" , "eq" , 1], - "blocked" : ["M" , "eq" , 2], - "blockedwake" : ["M" , "eq" , 3], - "locked" : ["M" , "eq" , 4], - "opcodeeq" : ["O" , "eq" , None], - "opcodelt" : ["O" , "lt" , None], - "opcodegt" : ["O" , "gt" , None], - "eventeq" : ["E" , "eq" , None], - "eventlt" : ["E" , "lt" , None], - "eventgt" : ["E" , "gt" , None], - } - -# Print usage information -def usage(): - print "rt-tester.py <-c -h -q -t> " - print " -c display comments after first command" - print " -h help" - print " -q quiet mode" - print " -t test mode (syntax check)" - print " testfile: read test specification from testfile" - print " otherwise from stdin" - return - -# Print progress when not in quiet mode -def progress(str): - if not quiet: - print str - -# Analyse a status value -def analyse(val, top, arg): - - intval = int(val) - - if top[0] == "M": - intval = intval / (10 ** int(arg)) - intval = intval % 10 - argval = top[2] - elif top[0] == "O": - argval = int(cmd_opcodes.get(arg, arg)) - else: - argval = int(arg) - - # progress("%d %s %d" %(intval, top[1], argval)) - - if top[1] == "eq" and intval == argval: - return 1 - if top[1] == "lt" and intval < argval: - return 1 - if top[1] == "gt" and intval > argval: - return 1 - return 0 - -# Parse the commandline -try: - (options, arguments) = getopt.getopt(sys.argv[1:],'chqt') -except getopt.GetoptError, ex: - usage() - sys.exit(1) - -# Parse commandline options -for option, value in options: - if option == "-c": - comments = 1 - elif option == "-q": - quiet = 1 - elif option == "-t": - test = 1 - elif option == '-h': - usage() - sys.exit(0) - -# Select the input source -if arguments: - try: - fd = open(arguments[0]) - except Exception,ex: - sys.stderr.write("File not found %s\n" %(arguments[0])) - sys.exit(1) -else: - fd = sys.stdin - -linenr = 0 - -# Read the test patterns -while 1: - - linenr = linenr + 1 - line = fd.readline() - if not len(line): - break - - line = line.strip() - parts = line.split(":") - - if not parts or len(parts) < 1: - continue - - if len(parts[0]) == 0: - continue - - if parts[0].startswith("#"): - if comments > 1: - progress(line) - continue - - if comments == 1: - comments = 2 - - progress(line) - - cmd = parts[0].strip().lower() - opc = parts[1].strip().lower() - tid = parts[2].strip() - dat = parts[3].strip() - - try: - # Test or wait for a status value - if cmd == "t" or cmd == "w": - testop = test_opcodes[opc] - - fname = "%s%s%s" %(sysfsprefix, tid, statusfile) - if test: - print fname - continue - - while 1: - query = 1 - fsta = open(fname, 'r') - status = fsta.readline().strip() - fsta.close() - stat = status.split(",") - for s in stat: - s = s.strip() - if s.startswith(testop[0]): - # Separate status value - val = s[2:].strip() - query = analyse(val, testop, dat) - break - if query or cmd == "t": - break - - progress(" " + status) - - if not query: - sys.stderr.write("Test failed in line %d\n" %(linenr)) - sys.exit(1) - - # Issue a command to the tester - elif cmd == "c": - cmdnr = cmd_opcodes[opc] - # Build command string and sys filename - cmdstr = "%s:%s" %(cmdnr, dat) - fname = "%s%s%s" %(sysfsprefix, tid, commandfile) - if test: - print fname - continue - fcmd = open(fname, 'w') - fcmd.write(cmdstr) - fcmd.close() - - except Exception,ex: - sys.stderr.write(str(ex)) - sys.stderr.write("\nSyntax error in line %d\n" %(linenr)) - if not test: - fd.close() - sys.exit(1) - -# Normal exit pass -print "Pass" -sys.exit(0) diff --git a/scripts/rt-tester/t2-l1-2rt-sameprio.tst b/scripts/rt-tester/t2-l1-2rt-sameprio.tst deleted file mode 100644 index 3710c8b2090d..000000000000 --- a/scripts/rt-tester/t2-l1-2rt-sameprio.tst +++ /dev/null @@ -1,94 +0,0 @@ -# -# RT-Mutex test -# -# Op: C(ommand)/T(est)/W(ait) -# | opcode -# | | threadid: 0-7 -# | | | opcode argument -# | | | | -# C: lock: 0: 0 -# -# Commands -# -# opcode opcode argument -# schedother nice value -# schedfifo priority -# lock lock nr (0-7) -# locknowait lock nr (0-7) -# lockint lock nr (0-7) -# lockintnowait lock nr (0-7) -# lockcont lock nr (0-7) -# unlock lock nr (0-7) -# signal 0 -# reset 0 -# resetevent 0 -# -# Tests / Wait -# -# opcode opcode argument -# -# prioeq priority -# priolt priority -# priogt priority -# nprioeq normal priority -# npriolt normal priority -# npriogt normal priority -# locked lock nr (0-7) -# blocked lock nr (0-7) -# blockedwake lock nr (0-7) -# unlocked lock nr (0-7) -# opcodeeq command opcode or number -# opcodelt number -# opcodegt number -# eventeq number -# eventgt number -# eventlt number - -# -# 2 threads 1 lock -# -C: resetevent: 0: 0 -W: opcodeeq: 0: 0 - -# Set schedulers -C: schedfifo: 0: 80 -C: schedfifo: 1: 80 - -# T0 lock L0 -C: locknowait: 0: 0 -C: locknowait: 1: 0 -W: locked: 0: 0 -W: blocked: 1: 0 -T: prioeq: 0: 80 - -# T0 unlock L0 -C: unlock: 0: 0 -W: locked: 1: 0 - -# Verify T0 -W: unlocked: 0: 0 -T: prioeq: 0: 80 - -# Unlock -C: unlock: 1: 0 -W: unlocked: 1: 0 - -# T1,T0 lock L0 -C: locknowait: 1: 0 -C: locknowait: 0: 0 -W: locked: 1: 0 -W: blocked: 0: 0 -T: prioeq: 1: 80 - -# T1 unlock L0 -C: unlock: 1: 0 -W: locked: 0: 0 - -# Verify T1 -W: unlocked: 1: 0 -T: prioeq: 1: 80 - -# Unlock and exit -C: unlock: 0: 0 -W: unlocked: 0: 0 - diff --git a/scripts/rt-tester/t2-l1-pi.tst b/scripts/rt-tester/t2-l1-pi.tst deleted file mode 100644 index b4cc95975adb..000000000000 --- a/scripts/rt-tester/t2-l1-pi.tst +++ /dev/null @@ -1,77 +0,0 @@ -# -# RT-Mutex test -# -# Op: C(ommand)/T(est)/W(ait) -# | opcode -# | | threadid: 0-7 -# | | | opcode argument -# | | | | -# C: lock: 0: 0 -# -# Commands -# -# opcode opcode argument -# schedother nice value -# schedfifo priority -# lock lock nr (0-7) -# locknowait lock nr (0-7) -# lockint lock nr (0-7) -# lockintnowait lock nr (0-7) -# lockcont lock nr (0-7) -# unlock lock nr (0-7) -# signal 0 -# reset 0 -# resetevent 0 -# -# Tests / Wait -# -# opcode opcode argument -# -# prioeq priority -# priolt priority -# priogt priority -# nprioeq normal priority -# npriolt normal priority -# npriogt normal priority -# locked lock nr (0-7) -# blocked lock nr (0-7) -# blockedwake lock nr (0-7) -# unlocked lock nr (0-7) -# opcodeeq command opcode or number -# opcodelt number -# opcodegt number -# eventeq number -# eventgt number -# eventlt number - -# -# 2 threads 1 lock with priority inversion -# -C: resetevent: 0: 0 -W: opcodeeq: 0: 0 - -# Set schedulers -C: schedother: 0: 0 -C: schedfifo: 1: 80 - -# T0 lock L0 -C: locknowait: 0: 0 -W: locked: 0: 0 - -# T1 lock L0 -C: locknowait: 1: 0 -W: blocked: 1: 0 -T: prioeq: 0: 80 - -# T0 unlock L0 -C: unlock: 0: 0 -W: locked: 1: 0 - -# Verify T1 -W: unlocked: 0: 0 -T: priolt: 0: 1 - -# Unlock and exit -C: unlock: 1: 0 -W: unlocked: 1: 0 - diff --git a/scripts/rt-tester/t2-l1-signal.tst b/scripts/rt-tester/t2-l1-signal.tst deleted file mode 100644 index 1b57376cc1f7..000000000000 --- a/scripts/rt-tester/t2-l1-signal.tst +++ /dev/null @@ -1,72 +0,0 @@ -# -# RT-Mutex test -# -# Op: C(ommand)/T(est)/W(ait) -# | opcode -# | | threadid: 0-7 -# | | | opcode argument -# | | | | -# C: lock: 0: 0 -# -# Commands -# -# opcode opcode argument -# schedother nice value -# schedfifo priority -# lock lock nr (0-7) -# locknowait lock nr (0-7) -# lockint lock nr (0-7) -# lockintnowait lock nr (0-7) -# lockcont lock nr (0-7) -# unlock lock nr (0-7) -# signal 0 -# reset 0 -# resetevent 0 -# -# Tests / Wait -# -# opcode opcode argument -# -# prioeq priority -# priolt priority -# priogt priority -# nprioeq normal priority -# npriolt normal priority -# npriogt normal priority -# locked lock nr (0-7) -# blocked lock nr (0-7) -# blockedwake lock nr (0-7) -# unlocked lock nr (0-7) -# opcodeeq command opcode or number -# opcodelt number -# opcodegt number -# eventeq number -# eventgt number -# eventlt number - -# -# 2 threads 1 lock with priority inversion -# -C: resetevent: 0: 0 -W: opcodeeq: 0: 0 - -# Set schedulers -C: schedother: 0: 0 -C: schedother: 1: 0 - -# T0 lock L0 -C: locknowait: 0: 0 -W: locked: 0: 0 - -# T1 lock L0 -C: lockintnowait: 1: 0 -W: blocked: 1: 0 - -# Interrupt T1 -C: signal: 1: 0 -W: unlocked: 1: 0 -T: opcodeeq: 1: -4 - -# Unlock and exit -C: unlock: 0: 0 -W: unlocked: 0: 0 diff --git a/scripts/rt-tester/t2-l2-2rt-deadlock.tst b/scripts/rt-tester/t2-l2-2rt-deadlock.tst deleted file mode 100644 index 68b10629b6f4..000000000000 --- a/scripts/rt-tester/t2-l2-2rt-deadlock.tst +++ /dev/null @@ -1,84 +0,0 @@ -# -# RT-Mutex test -# -# Op: C(ommand)/T(est)/W(ait) -# | opcode -# | | threadid: 0-7 -# | | | opcode argument -# | | | | -# C: lock: 0: 0 -# -# Commands -# -# opcode opcode argument -# schedother nice value -# schedfifo priority -# lock lock nr (0-7) -# locknowait lock nr (0-7) -# lockint lock nr (0-7) -# lockintnowait lock nr (0-7) -# lockcont lock nr (0-7) -# unlock lock nr (0-7) -# signal 0 -# reset 0 -# resetevent 0 -# -# Tests / Wait -# -# opcode opcode argument -# -# prioeq priority -# priolt priority -# priogt priority -# nprioeq normal priority -# npriolt normal priority -# npriogt normal priority -# locked lock nr (0-7) -# blocked lock nr (0-7) -# blockedwake lock nr (0-7) -# unlocked lock nr (0-7) -# opcodeeq command opcode or number -# opcodelt number -# opcodegt number -# eventeq number -# eventgt number -# eventlt number - -# -# 2 threads 2 lock -# -C: resetevent: 0: 0 -W: opcodeeq: 0: 0 - -# Set schedulers -C: schedfifo: 0: 80 -C: schedfifo: 1: 80 - -# T0 lock L0 -C: locknowait: 0: 0 -W: locked: 0: 0 - -# T1 lock L1 -C: locknowait: 1: 1 -W: locked: 1: 1 - -# T0 lock L1 -C: lockintnowait: 0: 1 -W: blocked: 0: 1 - -# T1 lock L0 -C: lockintnowait: 1: 0 -W: blocked: 1: 0 - -# Make deadlock go away -C: signal: 1: 0 -W: unlocked: 1: 0 -C: signal: 0: 0 -W: unlocked: 0: 1 - -# Unlock and exit -C: unlock: 0: 0 -W: unlocked: 0: 0 -C: unlock: 1: 1 -W: unlocked: 1: 1 - diff --git a/scripts/rt-tester/t3-l1-pi-1rt.tst b/scripts/rt-tester/t3-l1-pi-1rt.tst deleted file mode 100644 index 8e6c8b11ae56..000000000000 --- a/scripts/rt-tester/t3-l1-pi-1rt.tst +++ /dev/null @@ -1,87 +0,0 @@ -# -# rt-mutex test -# -# Op: C(ommand)/T(est)/W(ait) -# | opcode -# | | threadid: 0-7 -# | | | opcode argument -# | | | | -# C: lock: 0: 0 -# -# Commands -# -# opcode opcode argument -# schedother nice value -# schedfifo priority -# lock lock nr (0-7) -# locknowait lock nr (0-7) -# lockint lock nr (0-7) -# lockintnowait lock nr (0-7) -# lockcont lock nr (0-7) -# unlock lock nr (0-7) -# signal thread to signal (0-7) -# reset 0 -# resetevent 0 -# -# Tests / Wait -# -# opcode opcode argument -# -# prioeq priority -# priolt priority -# priogt priority -# nprioeq normal priority -# npriolt normal priority -# npriogt normal priority -# locked lock nr (0-7) -# blocked lock nr (0-7) -# blockedwake lock nr (0-7) -# unlocked lock nr (0-7) -# opcodeeq command opcode or number -# opcodelt number -# opcodegt number -# eventeq number -# eventgt number -# eventlt number - -# -# 3 threads 1 lock PI -# -C: resetevent: 0: 0 -W: opcodeeq: 0: 0 - -# Set schedulers -C: schedother: 0: 0 -C: schedother: 1: 0 -C: schedfifo: 2: 82 - -# T0 lock L0 -C: locknowait: 0: 0 -W: locked: 0: 0 - -# T1 lock L0 -C: locknowait: 1: 0 -W: blocked: 1: 0 -T: priolt: 0: 1 - -# T2 lock L0 -C: locknowait: 2: 0 -W: blocked: 2: 0 -T: prioeq: 0: 82 - -# T0 unlock L0 -C: unlock: 0: 0 - -# Wait until T2 got the lock -W: locked: 2: 0 -W: unlocked: 0: 0 -T: priolt: 0: 1 - -# T2 unlock L0 -C: unlock: 2: 0 - -W: unlocked: 2: 0 -W: locked: 1: 0 - -C: unlock: 1: 0 -W: unlocked: 1: 0 diff --git a/scripts/rt-tester/t3-l1-pi-2rt.tst b/scripts/rt-tester/t3-l1-pi-2rt.tst deleted file mode 100644 index 69c2212fc520..000000000000 --- a/scripts/rt-tester/t3-l1-pi-2rt.tst +++ /dev/null @@ -1,88 +0,0 @@ -# -# rt-mutex test -# -# Op: C(ommand)/T(est)/W(ait) -# | opcode -# | | threadid: 0-7 -# | | | opcode argument -# | | | | -# C: lock: 0: 0 -# -# Commands -# -# opcode opcode argument -# schedother nice value -# schedfifo priority -# lock lock nr (0-7) -# locknowait lock nr (0-7) -# lockint lock nr (0-7) -# lockintnowait lock nr (0-7) -# lockcont lock nr (0-7) -# unlock lock nr (0-7) -# signal thread to signal (0-7) -# reset 0 -# resetevent 0 -# -# Tests / Wait -# -# opcode opcode argument -# -# prioeq priority -# priolt priority -# priogt priority -# nprioeq normal priority -# npriolt normal priority -# npriogt normal priority -# locked lock nr (0-7) -# blocked lock nr (0-7) -# blockedwake lock nr (0-7) -# unlocked lock nr (0-7) -# opcodeeq command opcode or number -# opcodelt number -# opcodegt number -# eventeq number -# eventgt number -# eventlt number - -# -# 3 threads 1 lock PI -# -C: resetevent: 0: 0 -W: opcodeeq: 0: 0 - -# Set schedulers -C: schedother: 0: 0 -C: schedfifo: 1: 81 -C: schedfifo: 2: 82 - -# T0 lock L0 -C: locknowait: 0: 0 -W: locked: 0: 0 - -# T1 lock L0 -C: locknowait: 1: 0 -W: blocked: 1: 0 -T: prioeq: 0: 81 - -# T2 lock L0 -C: locknowait: 2: 0 -W: blocked: 2: 0 -T: prioeq: 0: 82 -T: prioeq: 1: 81 - -# T0 unlock L0 -C: unlock: 0: 0 - -# Wait until T2 got the lock -W: locked: 2: 0 -W: unlocked: 0: 0 -T: priolt: 0: 1 - -# T2 unlock L0 -C: unlock: 2: 0 - -W: unlocked: 2: 0 -W: locked: 1: 0 - -C: unlock: 1: 0 -W: unlocked: 1: 0 diff --git a/scripts/rt-tester/t3-l1-pi-3rt.tst b/scripts/rt-tester/t3-l1-pi-3rt.tst deleted file mode 100644 index 9b0f1eb26a88..000000000000 --- a/scripts/rt-tester/t3-l1-pi-3rt.tst +++ /dev/null @@ -1,87 +0,0 @@ -# -# rt-mutex test -# -# Op: C(ommand)/T(est)/W(ait) -# | opcode -# | | threadid: 0-7 -# | | | opcode argument -# | | | | -# C: lock: 0: 0 -# -# Commands -# -# opcode opcode argument -# schedother nice value -# schedfifo priority -# lock lock nr (0-7) -# locknowait lock nr (0-7) -# lockint lock nr (0-7) -# lockintnowait lock nr (0-7) -# lockcont lock nr (0-7) -# unlock lock nr (0-7) -# signal thread to signal (0-7) -# reset 0 -# resetevent 0 -# -# Tests / Wait -# -# opcode opcode argument -# -# prioeq priority -# priolt priority -# priogt priority -# nprioeq normal priority -# npriolt normal priority -# npriogt normal priority -# locked lock nr (0-7) -# blocked lock nr (0-7) -# blockedwake lock nr (0-7) -# unlocked lock nr (0-7) -# opcodeeq command opcode or number -# opcodelt number -# opcodegt number -# eventeq number -# eventgt number -# eventlt number - -# -# 3 threads 1 lock PI -# -C: resetevent: 0: 0 -W: opcodeeq: 0: 0 - -# Set schedulers -C: schedfifo: 0: 80 -C: schedfifo: 1: 81 -C: schedfifo: 2: 82 - -# T0 lock L0 -C: locknowait: 0: 0 -W: locked: 0: 0 - -# T1 lock L0 -C: locknowait: 1: 0 -W: blocked: 1: 0 -T: prioeq: 0: 81 - -# T2 lock L0 -C: locknowait: 2: 0 -W: blocked: 2: 0 -T: prioeq: 0: 82 - -# T0 unlock L0 -C: unlock: 0: 0 - -# Wait until T2 got the lock -W: locked: 2: 0 -W: unlocked: 0: 0 -T: prioeq: 0: 80 - -# T2 unlock L0 -C: unlock: 2: 0 - -W: locked: 1: 0 -W: unlocked: 2: 0 - -C: unlock: 1: 0 -W: unlocked: 1: 0 diff --git a/scripts/rt-tester/t3-l1-pi-signal.tst b/scripts/rt-tester/t3-l1-pi-signal.tst deleted file mode 100644 index 39ec74ab06ee..000000000000 --- a/scripts/rt-tester/t3-l1-pi-signal.tst +++ /dev/null @@ -1,93 +0,0 @@ -# -# rt-mutex test -# -# Op: C(ommand)/T(est)/W(ait) -# | opcode -# | | threadid: 0-7 -# | | | opcode argument -# | | | | -# C: lock: 0: 0 -# -# Commands -# -# opcode opcode argument -# schedother nice value -# schedfifo priority -# lock lock nr (0-7) -# locknowait lock nr (0-7) -# lockint lock nr (0-7) -# lockintnowait lock nr (0-7) -# lockcont lock nr (0-7) -# unlock lock nr (0-7) -# signal thread to signal (0-7) -# reset 0 -# resetevent 0 -# -# Tests / Wait -# -# opcode opcode argument -# -# prioeq priority -# priolt priority -# priogt priority -# nprioeq normal priority -# npriolt normal priority -# npriogt normal priority -# locked lock nr (0-7) -# blocked lock nr (0-7) -# blockedwake lock nr (0-7) -# unlocked lock nr (0-7) -# opcodeeq command opcode or number -# opcodelt number -# opcodegt number -# eventeq number -# eventgt number -# eventlt number - -# Reset event counter -C: resetevent: 0: 0 -W: opcodeeq: 0: 0 - -# Set priorities -C: schedother: 0: 0 -C: schedfifo: 1: 80 -C: schedfifo: 2: 81 - -# T0 lock L0 -C: lock: 0: 0 -W: locked: 0: 0 - -# T1 lock L0, no wait in the wakeup path -C: locknowait: 1: 0 -W: blocked: 1: 0 -T: prioeq: 0: 80 -T: prioeq: 1: 80 - -# T2 lock L0 interruptible, no wait in the wakeup path -C: lockintnowait: 2: 0 -W: blocked: 2: 0 -T: prioeq: 0: 81 -T: prioeq: 1: 80 - -# Interrupt T2 -C: signal: 2: 2 -W: unlocked: 2: 0 -T: prioeq: 1: 80 -T: prioeq: 0: 80 - -T: locked: 0: 0 -T: blocked: 1: 0 - -# T0 unlock L0 -C: unlock: 0: 0 - -# Wait until T1 has locked L0 and exit -W: locked: 1: 0 -W: unlocked: 0: 0 -T: priolt: 0: 1 - -C: unlock: 1: 0 -W: unlocked: 1: 0 - - - diff --git a/scripts/rt-tester/t3-l1-pi-steal.tst b/scripts/rt-tester/t3-l1-pi-steal.tst deleted file mode 100644 index e03db7e010fa..000000000000 --- a/scripts/rt-tester/t3-l1-pi-steal.tst +++ /dev/null @@ -1,91 +0,0 @@ -# -# rt-mutex test -# -# Op: C(ommand)/T(est)/W(ait) -# | opcode -# | | threadid: 0-7 -# | | | opcode argument -# | | | | -# C: lock: 0: 0 -# -# Commands -# -# opcode opcode argument -# schedother nice value -# schedfifo priority -# lock lock nr (0-7) -# locknowait lock nr (0-7) -# lockint lock nr (0-7) -# lockintnowait lock nr (0-7) -# lockcont lock nr (0-7) -# unlock lock nr (0-7) -# signal thread to signal (0-7) -# reset 0 -# resetevent 0 -# -# Tests / Wait -# -# opcode opcode argument -# -# prioeq priority -# priolt priority -# priogt priority -# nprioeq normal priority -# npriolt normal priority -# npriogt normal priority -# locked lock nr (0-7) -# blocked lock nr (0-7) -# blockedwake lock nr (0-7) -# unlocked lock nr (0-7) -# opcodeeq command opcode or number -# opcodelt number -# opcodegt number -# eventeq number -# eventgt number -# eventlt number - -# -# 3 threads 1 lock PI steal pending ownership -# -C: resetevent: 0: 0 -W: opcodeeq: 0: 0 - -# Set schedulers -C: schedother: 0: 0 -C: schedfifo: 1: 80 -C: schedfifo: 2: 81 - -# T0 lock L0 -C: lock: 0: 0 -W: locked: 0: 0 - -# T1 lock L0 -C: lock: 1: 0 -W: blocked: 1: 0 -T: prioeq: 0: 80 - -# T0 unlock L0 -C: unlock: 0: 0 - -# Wait until T1 is in the wakeup loop -W: blockedwake: 1: 0 -T: priolt: 0: 1 - -# T2 lock L0 -C: lock: 2: 0 -# T1 leave wakeup loop -C: lockcont: 1: 0 - -# T2 must have the lock and T1 must be blocked -W: locked: 2: 0 -W: blocked: 1: 0 - -# T2 unlock L0 -C: unlock: 2: 0 - -# Wait until T1 is in the wakeup loop and let it run -W: blockedwake: 1: 0 -C: lockcont: 1: 0 -W: locked: 1: 0 -C: unlock: 1: 0 -W: unlocked: 1: 0 diff --git a/scripts/rt-tester/t3-l2-pi.tst b/scripts/rt-tester/t3-l2-pi.tst deleted file mode 100644 index 7b59100d3e48..000000000000 --- a/scripts/rt-tester/t3-l2-pi.tst +++ /dev/null @@ -1,87 +0,0 @@ -# -# rt-mutex test -# -# Op: C(ommand)/T(est)/W(ait) -# | opcode -# | | threadid: 0-7 -# | | | opcode argument -# | | | | -# C: lock: 0: 0 -# -# Commands -# -# opcode opcode argument -# schedother nice value -# schedfifo priority -# lock lock nr (0-7) -# locknowait lock nr (0-7) -# lockint lock nr (0-7) -# lockintnowait lock nr (0-7) -# lockcont lock nr (0-7) -# unlock lock nr (0-7) -# signal thread to signal (0-7) -# reset 0 -# resetevent 0 -# -# Tests / Wait -# -# opcode opcode argument -# -# prioeq priority -# priolt priority -# priogt priority -# nprioeq normal priority -# npriolt normal priority -# npriogt normal priority -# locked lock nr (0-7) -# blocked lock nr (0-7) -# blockedwake lock nr (0-7) -# unlocked lock nr (0-7) -# opcodeeq command opcode or number -# opcodelt number -# opcodegt number -# eventeq number -# eventgt number -# eventlt number - -# -# 3 threads 2 lock PI -# -C: resetevent: 0: 0 -W: opcodeeq: 0: 0 - -# Set schedulers -C: schedother: 0: 0 -C: schedother: 1: 0 -C: schedfifo: 2: 82 - -# T0 lock L0 -C: locknowait: 0: 0 -W: locked: 0: 0 - -# T1 lock L0 -C: locknowait: 1: 0 -W: blocked: 1: 0 -T: priolt: 0: 1 - -# T2 lock L0 -C: locknowait: 2: 0 -W: blocked: 2: 0 -T: prioeq: 0: 82 - -# T0 unlock L0 -C: unlock: 0: 0 - -# Wait until T2 got the lock -W: locked: 2: 0 -W: unlocked: 0: 0 -T: priolt: 0: 1 - -# T2 unlock L0 -C: unlock: 2: 0 - -W: unlocked: 2: 0 -W: locked: 1: 0 - -C: unlock: 1: 0 -W: unlocked: 1: 0 diff --git a/scripts/rt-tester/t4-l2-pi-deboost.tst b/scripts/rt-tester/t4-l2-pi-deboost.tst deleted file mode 100644 index 2f0e049d6443..000000000000 --- a/scripts/rt-tester/t4-l2-pi-deboost.tst +++ /dev/null @@ -1,118 +0,0 @@ -# -# rt-mutex test -# -# Op: C(ommand)/T(est)/W(ait) -# | opcode -# | | threadid: 0-7 -# | | | opcode argument -# | | | | -# C: lock: 0: 0 -# -# Commands -# -# opcode opcode argument -# schedother nice value -# schedfifo priority -# lock lock nr (0-7) -# locknowait lock nr (0-7) -# lockint lock nr (0-7) -# lockintnowait lock nr (0-7) -# lockcont lock nr (0-7) -# unlock lock nr (0-7) -# signal thread to signal (0-7) -# reset 0 -# resetevent 0 -# -# Tests / Wait -# -# opcode opcode argument -# -# prioeq priority -# priolt priority -# priogt priority -# nprioeq normal priority -# npriolt normal priority -# npriogt normal priority -# locked lock nr (0-7) -# blocked lock nr (0-7) -# blockedwake lock nr (0-7) -# unlocked lock nr (0-7) -# opcodeeq command opcode or number -# opcodelt number -# opcodegt number -# eventeq number -# eventgt number -# eventlt number - -# -# 4 threads 2 lock PI -# -C: resetevent: 0: 0 -W: opcodeeq: 0: 0 - -# Set schedulers -C: schedother: 0: 0 -C: schedother: 1: 0 -C: schedfifo: 2: 82 -C: schedfifo: 3: 83 - -# T0 lock L0 -C: locknowait: 0: 0 -W: locked: 0: 0 - -# T1 lock L1 -C: locknowait: 1: 1 -W: locked: 1: 1 - -# T3 lock L0 -C: lockintnowait: 3: 0 -W: blocked: 3: 0 -T: prioeq: 0: 83 - -# T0 lock L1 -C: lock: 0: 1 -W: blocked: 0: 1 -T: prioeq: 1: 83 - -# T1 unlock L1 -C: unlock: 1: 1 - -# Wait until T0 is in the wakeup code -W: blockedwake: 0: 1 - -# Verify that T1 is unboosted -W: unlocked: 1: 1 -T: priolt: 1: 1 - -# T2 lock L1 (T0 is boosted and pending owner !) -C: locknowait: 2: 1 -W: blocked: 2: 1 -T: prioeq: 0: 83 - -# Interrupt T3 and wait until T3 returned -C: signal: 3: 0 -W: unlocked: 3: 0 - -# Verify prio of T0 (still pending owner, -# but T2 is enqueued due to the previous boost by T3 -T: prioeq: 0: 82 - -# Let T0 continue -C: lockcont: 0: 1 -W: locked: 0: 1 - -# Unlock L1 and let T2 get L1 -C: unlock: 0: 1 -W: locked: 2: 1 - -# Verify that T0 is unboosted -W: unlocked: 0: 1 -T: priolt: 0: 1 - -# Unlock everything and exit -C: unlock: 2: 1 -W: unlocked: 2: 1 - -C: unlock: 0: 0 -W: unlocked: 0: 0 - diff --git a/scripts/rt-tester/t5-l4-pi-boost-deboost-setsched.tst b/scripts/rt-tester/t5-l4-pi-boost-deboost-setsched.tst deleted file mode 100644 index 04f4034ff895..000000000000 --- a/scripts/rt-tester/t5-l4-pi-boost-deboost-setsched.tst +++ /dev/null @@ -1,178 +0,0 @@ -# -# rt-mutex test -# -# Op: C(ommand)/T(est)/W(ait) -# | opcode -# | | threadid: 0-7 -# | | | opcode argument -# | | | | -# C: lock: 0: 0 -# -# Commands -# -# opcode opcode argument -# schedother nice value -# schedfifo priority -# lock lock nr (0-7) -# locknowait lock nr (0-7) -# lockint lock nr (0-7) -# lockintnowait lock nr (0-7) -# lockcont lock nr (0-7) -# unlock lock nr (0-7) -# signal thread to signal (0-7) -# reset 0 -# resetevent 0 -# -# Tests / Wait -# -# opcode opcode argument -# -# prioeq priority -# priolt priority -# priogt priority -# nprioeq normal priority -# npriolt normal priority -# npriogt normal priority -# locked lock nr (0-7) -# blocked lock nr (0-7) -# blockedwake lock nr (0-7) -# unlocked lock nr (0-7) -# opcodeeq command opcode or number -# opcodelt number -# opcodegt number -# eventeq number -# eventgt number -# eventlt number - -# -# 5 threads 4 lock PI - modify priority of blocked threads -# -C: resetevent: 0: 0 -W: opcodeeq: 0: 0 - -# Set schedulers -C: schedother: 0: 0 -C: schedfifo: 1: 81 -C: schedfifo: 2: 82 -C: schedfifo: 3: 83 -C: schedfifo: 4: 84 - -# T0 lock L0 -C: locknowait: 0: 0 -W: locked: 0: 0 - -# T1 lock L1 -C: locknowait: 1: 1 -W: locked: 1: 1 - -# T1 lock L0 -C: lockintnowait: 1: 0 -W: blocked: 1: 0 -T: prioeq: 0: 81 - -# T2 lock L2 -C: locknowait: 2: 2 -W: locked: 2: 2 - -# T2 lock L1 -C: lockintnowait: 2: 1 -W: blocked: 2: 1 -T: prioeq: 0: 82 -T: prioeq: 1: 82 - -# T3 lock L3 -C: locknowait: 3: 3 -W: locked: 3: 3 - -# T3 lock L2 -C: lockintnowait: 3: 2 -W: blocked: 3: 2 -T: prioeq: 0: 83 -T: prioeq: 1: 83 -T: prioeq: 2: 83 - -# T4 lock L3 -C: lockintnowait: 4: 3 -W: blocked: 4: 3 -T: prioeq: 0: 84 -T: prioeq: 1: 84 -T: prioeq: 2: 84 -T: prioeq: 3: 84 - -# Reduce prio of T4 -C: schedfifo: 4: 80 -T: prioeq: 0: 83 -T: prioeq: 1: 83 -T: prioeq: 2: 83 -T: prioeq: 3: 83 -T: prioeq: 4: 80 - -# Increase prio of T4 -C: schedfifo: 4: 84 -T: prioeq: 0: 84 -T: prioeq: 1: 84 -T: prioeq: 2: 84 -T: prioeq: 3: 84 -T: prioeq: 4: 84 - -# Reduce prio of T3 -C: schedfifo: 3: 80 -T: prioeq: 0: 84 -T: prioeq: 1: 84 -T: prioeq: 2: 84 -T: prioeq: 3: 84 -T: prioeq: 4: 84 - -# Increase prio of T3 -C: schedfifo: 3: 85 -T: prioeq: 0: 85 -T: prioeq: 1: 85 -T: prioeq: 2: 85 -T: prioeq: 3: 85 -T: prioeq: 4: 84 - -# Reduce prio of T3 -C: schedfifo: 3: 83 -T: prioeq: 0: 84 -T: prioeq: 1: 84 -T: prioeq: 2: 84 -T: prioeq: 3: 84 -T: prioeq: 4: 84 - -# Signal T4 -C: signal: 4: 0 -W: unlocked: 4: 3 -T: prioeq: 0: 83 -T: prioeq: 1: 83 -T: prioeq: 2: 83 -T: prioeq: 3: 83 - -# Signal T3 -C: signal: 3: 0 -W: unlocked: 3: 2 -T: prioeq: 0: 82 -T: prioeq: 1: 82 -T: prioeq: 2: 82 - -# Signal T2 -C: signal: 2: 0 -W: unlocked: 2: 1 -T: prioeq: 0: 81 -T: prioeq: 1: 81 - -# Signal T1 -C: signal: 1: 0 -W: unlocked: 1: 0 -T: priolt: 0: 1 - -# Unlock and exit -C: unlock: 3: 3 -C: unlock: 2: 2 -C: unlock: 1: 1 -C: unlock: 0: 0 - -W: unlocked: 3: 3 -W: unlocked: 2: 2 -W: unlocked: 1: 1 -W: unlocked: 0: 0 - diff --git a/scripts/rt-tester/t5-l4-pi-boost-deboost.tst b/scripts/rt-tester/t5-l4-pi-boost-deboost.tst deleted file mode 100644 index a48a6ee29ddc..000000000000 --- a/scripts/rt-tester/t5-l4-pi-boost-deboost.tst +++ /dev/null @@ -1,138 +0,0 @@ -# -# rt-mutex test -# -# Op: C(ommand)/T(est)/W(ait) -# | opcode -# | | threadid: 0-7 -# | | | opcode argument -# | | | | -# C: lock: 0: 0 -# -# Commands -# -# opcode opcode argument -# schedother nice value -# schedfifo priority -# lock lock nr (0-7) -# locknowait lock nr (0-7) -# lockint lock nr (0-7) -# lockintnowait lock nr (0-7) -# lockcont lock nr (0-7) -# unlock lock nr (0-7) -# signal thread to signal (0-7) -# reset 0 -# resetevent 0 -# -# Tests / Wait -# -# opcode opcode argument -# -# prioeq priority -# priolt priority -# priogt priority -# nprioeq normal priority -# npriolt normal priority -# npriogt normal priority -# locked lock nr (0-7) -# blocked lock nr (0-7) -# blockedwake lock nr (0-7) -# unlocked lock nr (0-7) -# opcodeeq command opcode or number -# opcodelt number -# opcodegt number -# eventeq number -# eventgt number -# eventlt number - -# -# 5 threads 4 lock PI -# -C: resetevent: 0: 0 -W: opcodeeq: 0: 0 - -# Set schedulers -C: schedother: 0: 0 -C: schedfifo: 1: 81 -C: schedfifo: 2: 82 -C: schedfifo: 3: 83 -C: schedfifo: 4: 84 - -# T0 lock L0 -C: locknowait: 0: 0 -W: locked: 0: 0 - -# T1 lock L1 -C: locknowait: 1: 1 -W: locked: 1: 1 - -# T1 lock L0 -C: lockintnowait: 1: 0 -W: blocked: 1: 0 -T: prioeq: 0: 81 - -# T2 lock L2 -C: locknowait: 2: 2 -W: locked: 2: 2 - -# T2 lock L1 -C: lockintnowait: 2: 1 -W: blocked: 2: 1 -T: prioeq: 0: 82 -T: prioeq: 1: 82 - -# T3 lock L3 -C: locknowait: 3: 3 -W: locked: 3: 3 - -# T3 lock L2 -C: lockintnowait: 3: 2 -W: blocked: 3: 2 -T: prioeq: 0: 83 -T: prioeq: 1: 83 -T: prioeq: 2: 83 - -# T4 lock L3 -C: lockintnowait: 4: 3 -W: blocked: 4: 3 -T: prioeq: 0: 84 -T: prioeq: 1: 84 -T: prioeq: 2: 84 -T: prioeq: 3: 84 - -# Signal T4 -C: signal: 4: 0 -W: unlocked: 4: 3 -T: prioeq: 0: 83 -T: prioeq: 1: 83 -T: prioeq: 2: 83 -T: prioeq: 3: 83 - -# Signal T3 -C: signal: 3: 0 -W: unlocked: 3: 2 -T: prioeq: 0: 82 -T: prioeq: 1: 82 -T: prioeq: 2: 82 - -# Signal T2 -C: signal: 2: 0 -W: unlocked: 2: 1 -T: prioeq: 0: 81 -T: prioeq: 1: 81 - -# Signal T1 -C: signal: 1: 0 -W: unlocked: 1: 0 -T: priolt: 0: 1 - -# Unlock and exit -C: unlock: 3: 3 -C: unlock: 2: 2 -C: unlock: 1: 1 -C: unlock: 0: 0 - -W: unlocked: 3: 3 -W: unlocked: 2: 2 -W: unlocked: 1: 1 -W: unlocked: 0: 0 - -- cgit v1.2.3 From 212d3be102d73dce70cc12f39dce4e0aed2c025b Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 20:07:47 +0200 Subject: alpha: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/alpha/include/asm/atomic.h | 43 +++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 15 deletions(-) (limited to 'arch') diff --git a/arch/alpha/include/asm/atomic.h b/arch/alpha/include/asm/atomic.h index 8f8eafbedd7c..0eff853398d2 100644 --- a/arch/alpha/include/asm/atomic.h +++ b/arch/alpha/include/asm/atomic.h @@ -29,13 +29,13 @@ * branch back to restart the operation. */ -#define ATOMIC_OP(op) \ +#define ATOMIC_OP(op, asm_op) \ static __inline__ void atomic_##op(int i, atomic_t * v) \ { \ unsigned long temp; \ __asm__ __volatile__( \ "1: ldl_l %0,%1\n" \ - " " #op "l %0,%2,%0\n" \ + " " #asm_op " %0,%2,%0\n" \ " stl_c %0,%1\n" \ " beq %0,2f\n" \ ".subsection 2\n" \ @@ -45,15 +45,15 @@ static __inline__ void atomic_##op(int i, atomic_t * v) \ :"Ir" (i), "m" (v->counter)); \ } \ -#define ATOMIC_OP_RETURN(op) \ +#define ATOMIC_OP_RETURN(op, asm_op) \ static inline int atomic_##op##_return(int i, atomic_t *v) \ { \ long temp, result; \ smp_mb(); \ __asm__ __volatile__( \ "1: ldl_l %0,%1\n" \ - " " #op "l %0,%3,%2\n" \ - " " #op "l %0,%3,%0\n" \ + " " #asm_op " %0,%3,%2\n" \ + " " #asm_op " %0,%3,%0\n" \ " stl_c %0,%1\n" \ " beq %0,2f\n" \ ".subsection 2\n" \ @@ -65,13 +65,13 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ return result; \ } -#define ATOMIC64_OP(op) \ +#define ATOMIC64_OP(op, asm_op) \ static __inline__ void atomic64_##op(long i, atomic64_t * v) \ { \ unsigned long temp; \ __asm__ __volatile__( \ "1: ldq_l %0,%1\n" \ - " " #op "q %0,%2,%0\n" \ + " " #asm_op " %0,%2,%0\n" \ " stq_c %0,%1\n" \ " beq %0,2f\n" \ ".subsection 2\n" \ @@ -81,15 +81,15 @@ static __inline__ void atomic64_##op(long i, atomic64_t * v) \ :"Ir" (i), "m" (v->counter)); \ } \ -#define ATOMIC64_OP_RETURN(op) \ +#define ATOMIC64_OP_RETURN(op, asm_op) \ static __inline__ long atomic64_##op##_return(long i, atomic64_t * v) \ { \ long temp, result; \ smp_mb(); \ __asm__ __volatile__( \ "1: ldq_l %0,%1\n" \ - " " #op "q %0,%3,%2\n" \ - " " #op "q %0,%3,%0\n" \ + " " #asm_op " %0,%3,%2\n" \ + " " #asm_op " %0,%3,%0\n" \ " stq_c %0,%1\n" \ " beq %0,2f\n" \ ".subsection 2\n" \ @@ -101,15 +101,28 @@ static __inline__ long atomic64_##op##_return(long i, atomic64_t * v) \ return result; \ } -#define ATOMIC_OPS(opg) \ - ATOMIC_OP(opg) \ - ATOMIC_OP_RETURN(opg) \ - ATOMIC64_OP(opg) \ - ATOMIC64_OP_RETURN(opg) +#define ATOMIC_OPS(op) \ + ATOMIC_OP(op, op##l) \ + ATOMIC_OP_RETURN(op, op##l) \ + ATOMIC64_OP(op, op##q) \ + ATOMIC64_OP_RETURN(op, op##q) ATOMIC_OPS(add) ATOMIC_OPS(sub) +#define CONFIG_ARCH_HAS_ATOMIC_OR +#define atomic_andnot atomic_andnot +#define atomic64_andnot atomic64_andnot + +ATOMIC_OP(and, and) +ATOMIC_OP(andnot, bic) +ATOMIC_OP(or, bis) +ATOMIC_OP(xor, xor) +ATOMIC64_OP(and, and) +ATOMIC64_OP(andnot, bic) +ATOMIC64_OP(or, bis) +ATOMIC64_OP(xor, xor) + #undef ATOMIC_OPS #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP -- cgit v1.2.3 From cda7e4137a4e39c7d877b6ed41da13a5b30dd2cf Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 20:06:20 +0200 Subject: arc: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Acked-by: Vineet Gupta Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/arc/include/asm/atomic.h | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'arch') diff --git a/arch/arc/include/asm/atomic.h b/arch/arc/include/asm/atomic.h index 03484cb4d16d..e90b701fc6a8 100644 --- a/arch/arc/include/asm/atomic.h +++ b/arch/arc/include/asm/atomic.h @@ -143,14 +143,29 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add, +=, add) ATOMIC_OPS(sub, -=, sub) -ATOMIC_OP(and, &=, and) -#define atomic_clear_mask(mask, v) atomic_and(~(mask), (v)) +#define CONFIG_ARCH_HAS_ATOMIC_OR +#define atomic_andnot atomic_andnot + +ATOMIC_OP(and, &=, and) +ATOMIC_OP(andnot, &= ~, bic) +ATOMIC_OP(or, |=, or) +ATOMIC_OP(xor, ^=, xor) #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP +static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) +{ + atomic_and(~mask, v); +} + +static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) +{ + atomic_or(mask, v); +} + /** * __atomic_add_unless - add unless the number is a given value * @v: pointer of type atomic_t -- cgit v1.2.3 From 125897908c718972351b589da89b7f990892d4df Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 20:04:39 +0200 Subject: arm: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/arm/include/asm/atomic.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'arch') diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h index e22c11970b7b..ff214bac9cb4 100644 --- a/arch/arm/include/asm/atomic.h +++ b/arch/arm/include/asm/atomic.h @@ -194,6 +194,14 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) ATOMIC_OPS(add, +=, add) ATOMIC_OPS(sub, -=, sub) +#define CONFIG_ARCH_HAS_ATOMIC_OR +#define atomic_andnot atomic_andnot + +ATOMIC_OP(and, &=, and) +ATOMIC_OP(andnot, &= ~, bic) +ATOMIC_OP(or, |=, orr) +ATOMIC_OP(xor, ^=, eor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -321,6 +329,13 @@ static inline long long atomic64_##op##_return(long long i, atomic64_t *v) \ ATOMIC64_OPS(add, adds, adc) ATOMIC64_OPS(sub, subs, sbc) +#define atomic64_andnot atomic64_andnot + +ATOMIC64_OP(and, and, and) +ATOMIC64_OP(andnot, bic, bic) +ATOMIC64_OP(or, orr, orr) +ATOMIC64_OP(xor, eor, eor) + #undef ATOMIC64_OPS #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP -- cgit v1.2.3 From 22288b40e2c7ab484a08a03df32782a67903912c Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 20:03:13 +0200 Subject: arm64: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/arm64/include/asm/atomic.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'arch') diff --git a/arch/arm64/include/asm/atomic.h b/arch/arm64/include/asm/atomic.h index 7047051ded40..2876173397b2 100644 --- a/arch/arm64/include/asm/atomic.h +++ b/arch/arm64/include/asm/atomic.h @@ -85,6 +85,14 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add, add) ATOMIC_OPS(sub, sub) +#define CONFIG_ARCH_HAS_ATOMIC_OR +#define atomic_andnot atomic_andnot + +ATOMIC_OP(and, and) +ATOMIC_OP(andnot, bic) +ATOMIC_OP(or, orr) +ATOMIC_OP(xor, eor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -183,6 +191,13 @@ static inline long atomic64_##op##_return(long i, atomic64_t *v) \ ATOMIC64_OPS(add, add) ATOMIC64_OPS(sub, sub) +#define atomic64_andnot atomic64_andnot + +ATOMIC64_OP(and, and) +ATOMIC64_OP(andnot, bic) +ATOMIC64_OP(or, orr) +ATOMIC64_OP(xor, eor) + #undef ATOMIC64_OPS #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP -- cgit v1.2.3 From f8a570e270bf62363cd498ac2ac8ea07a76ad4d6 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 20:02:20 +0200 Subject: avr32: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/avr32/include/asm/atomic.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'arch') diff --git a/arch/avr32/include/asm/atomic.h b/arch/avr32/include/asm/atomic.h index 2d07ce1c5327..115d3005e4bc 100644 --- a/arch/avr32/include/asm/atomic.h +++ b/arch/avr32/include/asm/atomic.h @@ -44,6 +44,20 @@ static inline int __atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OP_RETURN(sub, sub, rKs21) ATOMIC_OP_RETURN(add, add, r) +#define ATOMIC_OP(op, asm_op) \ +ATOMIC_OP_RETURN(op, asm_op, r) \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + (void)__atomic_##op##_return(i, v); \ +} + +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC_OP(and, and) +ATOMIC_OP(or, or) +ATOMIC_OP(xor, eor) + +#undef ATOMIC_OP #undef ATOMIC_OP_RETURN /* -- cgit v1.2.3 From d835b6c4cc02507b3bf3f8ee6c86857cf0ee67ab Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 23 Apr 2015 21:44:42 +0200 Subject: blackfin: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. TODO: use inline asm or at least asm macros to collapse the lot. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/blackfin/include/asm/atomic.h | 28 +++++++++++++++++++++------- arch/blackfin/kernel/bfin_ksyms.c | 7 ++++--- arch/blackfin/mach-bf561/atomic.S | 30 +++++++++++++++--------------- 3 files changed, 40 insertions(+), 25 deletions(-) (limited to 'arch') diff --git a/arch/blackfin/include/asm/atomic.h b/arch/blackfin/include/asm/atomic.h index a107a98e9978..eafa55b81a7b 100644 --- a/arch/blackfin/include/asm/atomic.h +++ b/arch/blackfin/include/asm/atomic.h @@ -16,19 +16,33 @@ #include asmlinkage int __raw_uncached_fetch_asm(const volatile int *ptr); -asmlinkage int __raw_atomic_update_asm(volatile int *ptr, int value); -asmlinkage int __raw_atomic_clear_asm(volatile int *ptr, int value); -asmlinkage int __raw_atomic_set_asm(volatile int *ptr, int value); +asmlinkage int __raw_atomic_add_asm(volatile int *ptr, int value); + +asmlinkage int __raw_atomic_and_asm(volatile int *ptr, int value); +asmlinkage int __raw_atomic_or_asm(volatile int *ptr, int value); asmlinkage int __raw_atomic_xor_asm(volatile int *ptr, int value); asmlinkage int __raw_atomic_test_asm(const volatile int *ptr, int value); #define atomic_read(v) __raw_uncached_fetch_asm(&(v)->counter) -#define atomic_add_return(i, v) __raw_atomic_update_asm(&(v)->counter, i) -#define atomic_sub_return(i, v) __raw_atomic_update_asm(&(v)->counter, -(i)) +#define atomic_add_return(i, v) __raw_atomic_add_asm(&(v)->counter, i) +#define atomic_sub_return(i, v) __raw_atomic_add_asm(&(v)->counter, -(i)) + +#define CONFIG_ARCH_HAS_ATOMIC_OR + +#define atomic_or(i, v) (void)__raw_atomic_or_asm(&(v)->counter, i) +#define atomic_and(i, v) (void)__raw_atomic_and_asm(&(v)->counter, i) +#define atomic_xor(i, v) (void)__raw_atomic_xor_asm(&(v)->counter, i) + +static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) +{ + atomic_and(~mask, v); +} -#define atomic_clear_mask(m, v) __raw_atomic_clear_asm(&(v)->counter, m) -#define atomic_set_mask(m, v) __raw_atomic_set_asm(&(v)->counter, m) +static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) +{ + atomic_or(mask, v); +} #endif diff --git a/arch/blackfin/kernel/bfin_ksyms.c b/arch/blackfin/kernel/bfin_ksyms.c index c446591b961d..a401c27b69b4 100644 --- a/arch/blackfin/kernel/bfin_ksyms.c +++ b/arch/blackfin/kernel/bfin_ksyms.c @@ -83,11 +83,12 @@ EXPORT_SYMBOL(insl); EXPORT_SYMBOL(insl_16); #ifdef CONFIG_SMP -EXPORT_SYMBOL(__raw_atomic_update_asm); -EXPORT_SYMBOL(__raw_atomic_clear_asm); -EXPORT_SYMBOL(__raw_atomic_set_asm); +EXPORT_SYMBOL(__raw_atomic_add_asm); +EXPORT_SYMBOL(__raw_atomic_and_asm); +EXPORT_SYMBOL(__raw_atomic_or_asm); EXPORT_SYMBOL(__raw_atomic_xor_asm); EXPORT_SYMBOL(__raw_atomic_test_asm); + EXPORT_SYMBOL(__raw_xchg_1_asm); EXPORT_SYMBOL(__raw_xchg_2_asm); EXPORT_SYMBOL(__raw_xchg_4_asm); diff --git a/arch/blackfin/mach-bf561/atomic.S b/arch/blackfin/mach-bf561/atomic.S index 2a08df8e8c4c..26fccb5568b9 100644 --- a/arch/blackfin/mach-bf561/atomic.S +++ b/arch/blackfin/mach-bf561/atomic.S @@ -587,10 +587,10 @@ ENDPROC(___raw_write_unlock_asm) * r0 = ptr * r1 = value * - * Add a signed value to a 32bit word and return the new value atomically. + * ADD a signed value to a 32bit word and return the new value atomically. * Clobbers: r3:0, p1:0 */ -ENTRY(___raw_atomic_update_asm) +ENTRY(___raw_atomic_add_asm) p1 = r0; r3 = r1; [--sp] = rets; @@ -603,19 +603,19 @@ ENTRY(___raw_atomic_update_asm) r0 = r3; rets = [sp++]; rts; -ENDPROC(___raw_atomic_update_asm) +ENDPROC(___raw_atomic_add_asm) /* * r0 = ptr * r1 = mask * - * Clear the mask bits from a 32bit word and return the old 32bit value + * AND the mask bits from a 32bit word and return the old 32bit value * atomically. * Clobbers: r3:0, p1:0 */ -ENTRY(___raw_atomic_clear_asm) +ENTRY(___raw_atomic_and_asm) p1 = r0; - r3 = ~r1; + r3 = r1; [--sp] = rets; call _get_core_lock; r2 = [p1]; @@ -627,17 +627,17 @@ ENTRY(___raw_atomic_clear_asm) r0 = r3; rets = [sp++]; rts; -ENDPROC(___raw_atomic_clear_asm) +ENDPROC(___raw_atomic_and_asm) /* * r0 = ptr * r1 = mask * - * Set the mask bits into a 32bit word and return the old 32bit value + * OR the mask bits into a 32bit word and return the old 32bit value * atomically. * Clobbers: r3:0, p1:0 */ -ENTRY(___raw_atomic_set_asm) +ENTRY(___raw_atomic_or_asm) p1 = r0; r3 = r1; [--sp] = rets; @@ -651,7 +651,7 @@ ENTRY(___raw_atomic_set_asm) r0 = r3; rets = [sp++]; rts; -ENDPROC(___raw_atomic_set_asm) +ENDPROC(___raw_atomic_or_asm) /* * r0 = ptr @@ -787,7 +787,7 @@ ENTRY(___raw_bit_set_asm) r2 = r1; r1 = 1; r1 <<= r2; - jump ___raw_atomic_set_asm + jump ___raw_atomic_or_asm ENDPROC(___raw_bit_set_asm) /* @@ -798,10 +798,10 @@ ENDPROC(___raw_bit_set_asm) * Clobbers: r3:0, p1:0 */ ENTRY(___raw_bit_clear_asm) - r2 = r1; - r1 = 1; - r1 <<= r2; - jump ___raw_atomic_clear_asm + r2 = 1; + r2 <<= r1; + r1 = ~r2; + jump ___raw_atomic_and_asm ENDPROC(___raw_bit_clear_asm) /* -- cgit v1.2.3 From 610f7ba93b1e4cda7370f367a6042cbda61227bb Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 20:00:54 +0200 Subject: hexagon: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/hexagon/include/asm/atomic.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'arch') diff --git a/arch/hexagon/include/asm/atomic.h b/arch/hexagon/include/asm/atomic.h index 93d07025f183..4efe2c7c0dd8 100644 --- a/arch/hexagon/include/asm/atomic.h +++ b/arch/hexagon/include/asm/atomic.h @@ -132,6 +132,12 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -- cgit v1.2.3 From 70ed47390d10a7351e44c3a07c5de376780cb6c1 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 20:00:01 +0200 Subject: ia64: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/ia64/include/asm/atomic.h | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/ia64/include/asm/atomic.h b/arch/ia64/include/asm/atomic.h index 0bf03501fe5c..0809ef5d6b9a 100644 --- a/arch/ia64/include/asm/atomic.h +++ b/arch/ia64/include/asm/atomic.h @@ -45,8 +45,6 @@ ia64_atomic_##op (int i, atomic_t *v) \ ATOMIC_OP(add, +) ATOMIC_OP(sub, -) -#undef ATOMIC_OP - #define atomic_add_return(i,v) \ ({ \ int __ia64_aar_i = (i); \ @@ -71,6 +69,18 @@ ATOMIC_OP(sub, -) : ia64_atomic_sub(__ia64_asr_i, v); \ }) +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC_OP(and, &) +ATOMIC_OP(or, |) +ATOMIC_OP(xor, ^) + +#define atomic_and(i,v) (void)ia64_atomic_and(i,v) +#define atomic_or(i,v) (void)ia64_atomic_or(i,v) +#define atomic_xor(i,v) (void)ia64_atomic_xor(i,v) + +#undef ATOMIC_OP + #define ATOMIC64_OP(op, c_op) \ static __inline__ long \ ia64_atomic64_##op (__s64 i, atomic64_t *v) \ @@ -89,8 +99,6 @@ ia64_atomic64_##op (__s64 i, atomic64_t *v) \ ATOMIC64_OP(add, +) ATOMIC64_OP(sub, -) -#undef ATOMIC64_OP - #define atomic64_add_return(i,v) \ ({ \ long __ia64_aar_i = (i); \ @@ -115,6 +123,16 @@ ATOMIC64_OP(sub, -) : ia64_atomic64_sub(__ia64_asr_i, v); \ }) +ATOMIC64_OP(and, &) +ATOMIC64_OP(or, |) +ATOMIC64_OP(xor, ^) + +#define atomic64_and(i,v) (void)ia64_atomic64_and(i,v) +#define atomic64_or(i,v) (void)ia64_atomic64_or(i,v) +#define atomic64_xor(i,v) (void)ia64_atomic64_xor(i,v) + +#undef ATOMIC64_OP + #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) -- cgit v1.2.3 From c66e45edef51e1f54297ddaf202fc2dd00852734 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 19:57:49 +0200 Subject: m32r: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/m32r/include/asm/atomic.h | 44 ++++++++++-------------------------------- 1 file changed, 10 insertions(+), 34 deletions(-) (limited to 'arch') diff --git a/arch/m32r/include/asm/atomic.h b/arch/m32r/include/asm/atomic.h index 31bb74adba08..7245463c1e98 100644 --- a/arch/m32r/include/asm/atomic.h +++ b/arch/m32r/include/asm/atomic.h @@ -94,6 +94,12 @@ static __inline__ int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -240,44 +246,14 @@ static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) } -static __inline__ void atomic_clear_mask(unsigned long mask, atomic_t *addr) +static __inline__ __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) { - unsigned long flags; - unsigned long tmp; - - local_irq_save(flags); - __asm__ __volatile__ ( - "# atomic_clear_mask \n\t" - DCACHE_CLEAR("%0", "r5", "%1") - M32R_LOCK" %0, @%1; \n\t" - "and %0, %2; \n\t" - M32R_UNLOCK" %0, @%1; \n\t" - : "=&r" (tmp) - : "r" (addr), "r" (~mask) - : "memory" - __ATOMIC_CLOBBER - ); - local_irq_restore(flags); + atomic_and(~mask, v); } -static __inline__ void atomic_set_mask(unsigned long mask, atomic_t *addr) +static __inline__ __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) { - unsigned long flags; - unsigned long tmp; - - local_irq_save(flags); - __asm__ __volatile__ ( - "# atomic_set_mask \n\t" - DCACHE_CLEAR("%0", "r5", "%1") - M32R_LOCK" %0, @%1; \n\t" - "or %0, %2; \n\t" - M32R_UNLOCK" %0, @%1; \n\t" - : "=&r" (tmp) - : "r" (addr), "r" (mask) - : "memory" - __ATOMIC_CLOBBER - ); - local_irq_restore(flags); + atomic_or(mask, v); } #endif /* _ASM_M32R_ATOMIC_H */ -- cgit v1.2.3 From 74b1bc505b058efdfef33e775df68d8324bbf8b8 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 19:56:20 +0200 Subject: m68k: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Acked-by: Geert Uytterhoeven Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/m68k/include/asm/atomic.h | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/m68k/include/asm/atomic.h b/arch/m68k/include/asm/atomic.h index e85f047fb072..c30e43ea49a3 100644 --- a/arch/m68k/include/asm/atomic.h +++ b/arch/m68k/include/asm/atomic.h @@ -77,6 +77,12 @@ static inline int atomic_##op##_return(int i, atomic_t * v) \ ATOMIC_OPS(add, +=, add) ATOMIC_OPS(sub, -=, sub) +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC_OP(and, &=, and) +ATOMIC_OP(or, |=, or) +ATOMIC_OP(xor, ^=, eor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -170,14 +176,14 @@ static inline int atomic_add_negative(int i, atomic_t *v) return c != 0; } -static inline void atomic_clear_mask(unsigned long mask, unsigned long *v) +static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) { - __asm__ __volatile__("andl %1,%0" : "+m" (*v) : ASM_DI (~(mask))); + atomic_and(~mask, v); } -static inline void atomic_set_mask(unsigned long mask, unsigned long *v) +static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) { - __asm__ __volatile__("orl %1,%0" : "+m" (*v) : ASM_DI (mask)); + atomic_or(mask, v); } static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) -- cgit v1.2.3 From 5b4a2f0f91fd4a635f380f847254d23121972e7b Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 19:53:39 +0200 Subject: metag: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/metag/include/asm/atomic_lnkget.h | 38 +++++++++------------------------- arch/metag/include/asm/atomic_lock1.h | 21 +++++++------------ 2 files changed, 17 insertions(+), 42 deletions(-) (limited to 'arch') diff --git a/arch/metag/include/asm/atomic_lnkget.h b/arch/metag/include/asm/atomic_lnkget.h index 948d8688643c..930c12cb8d37 100644 --- a/arch/metag/include/asm/atomic_lnkget.h +++ b/arch/metag/include/asm/atomic_lnkget.h @@ -74,42 +74,24 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) +static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) { - int temp; - - asm volatile ( - "1: LNKGETD %0, [%1]\n" - " AND %0, %0, %2\n" - " LNKSETD [%1] %0\n" - " DEFR %0, TXSTAT\n" - " ANDT %0, %0, #HI(0x3f000000)\n" - " CMPT %0, #HI(0x02000000)\n" - " BNZ 1b\n" - : "=&d" (temp) - : "da" (&v->counter), "bd" (~mask) - : "cc"); + atomic_and(~mask, v); } -static inline void atomic_set_mask(unsigned int mask, atomic_t *v) +static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) { - int temp; - - asm volatile ( - "1: LNKGETD %0, [%1]\n" - " OR %0, %0, %2\n" - " LNKSETD [%1], %0\n" - " DEFR %0, TXSTAT\n" - " ANDT %0, %0, #HI(0x3f000000)\n" - " CMPT %0, #HI(0x02000000)\n" - " BNZ 1b\n" - : "=&d" (temp) - : "da" (&v->counter), "bd" (mask) - : "cc"); + atomic_or(mask, v); } static inline int atomic_cmpxchg(atomic_t *v, int old, int new) diff --git a/arch/metag/include/asm/atomic_lock1.h b/arch/metag/include/asm/atomic_lock1.h index f5d5898c1020..7d88725a85da 100644 --- a/arch/metag/include/asm/atomic_lock1.h +++ b/arch/metag/include/asm/atomic_lock1.h @@ -68,29 +68,22 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add, +=) ATOMIC_OPS(sub, -=) +ATOMIC_OP(and, &=) +ATOMIC_OP(or, |=) +ATOMIC_OP(xor, ^=) #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) +static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) { - unsigned long flags; - - __global_lock1(flags); - fence(); - v->counter &= ~mask; - __global_unlock1(flags); + atomic_and(~mask, v); } -static inline void atomic_set_mask(unsigned int mask, atomic_t *v) +static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) { - unsigned long flags; - - __global_lock1(flags); - fence(); - v->counter |= mask; - __global_unlock1(flags); + atomic_or(mask, v); } static inline int atomic_cmpxchg(atomic_t *v, int old, int new) -- cgit v1.2.3 From 27782f2752aca65a241f10fb2d4508c71bb2656b Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 19:51:36 +0200 Subject: mips: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Acked-by: Ralf Baechle Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/mips/include/asm/atomic.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'arch') diff --git a/arch/mips/include/asm/atomic.h b/arch/mips/include/asm/atomic.h index 26d436336f2e..0430ba6ab762 100644 --- a/arch/mips/include/asm/atomic.h +++ b/arch/mips/include/asm/atomic.h @@ -137,6 +137,12 @@ static __inline__ int atomic_##op##_return(int i, atomic_t * v) \ ATOMIC_OPS(add, +=, addu) ATOMIC_OPS(sub, -=, subu) +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC_OP(and, &=, and) +ATOMIC_OP(or, |=, or) +ATOMIC_OP(xor, ^=, xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -416,6 +422,9 @@ static __inline__ long atomic64_##op##_return(long i, atomic64_t * v) \ ATOMIC64_OPS(add, +=, daddu) ATOMIC64_OPS(sub, -=, dsubu) +ATOMIC64_OP(and, &=, and) +ATOMIC64_OP(or, |=, or) +ATOMIC64_OP(xor, ^=, xor) #undef ATOMIC64_OPS #undef ATOMIC64_OP_RETURN -- cgit v1.2.3 From ddb7573ff68964e7b3b72eeb9cde1384c4c6ba83 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 19:50:20 +0200 Subject: mn10300: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/mn10300/include/asm/atomic.h | 57 +++++++-------------------------------- 1 file changed, 10 insertions(+), 47 deletions(-) (limited to 'arch') diff --git a/arch/mn10300/include/asm/atomic.h b/arch/mn10300/include/asm/atomic.h index 5be655e83e70..03eea8158cf9 100644 --- a/arch/mn10300/include/asm/atomic.h +++ b/arch/mn10300/include/asm/atomic.h @@ -89,6 +89,12 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -134,31 +140,9 @@ static inline void atomic_dec(atomic_t *v) * * Atomically clears the bits set in mask from the memory word specified. */ -static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr) +static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) { -#ifdef CONFIG_SMP - int status; - - asm volatile( - "1: mov %3,(_AAR,%2) \n" - " mov (_ADR,%2),%0 \n" - " and %4,%0 \n" - " mov %0,(_ADR,%2) \n" - " mov (_ADR,%2),%0 \n" /* flush */ - " mov (_ASR,%2),%0 \n" - " or %0,%0 \n" - " bne 1b \n" - : "=&r"(status), "=m"(*addr) - : "a"(ATOMIC_OPS_BASE_ADDR), "r"(addr), "r"(~mask) - : "memory", "cc"); -#else - unsigned long flags; - - mask = ~mask; - flags = arch_local_cli_save(); - *addr &= mask; - arch_local_irq_restore(flags); -#endif + atomic_and(~mask, v); } /** @@ -168,30 +152,9 @@ static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr) * * Atomically sets the bits set in mask from the memory word specified. */ -static inline void atomic_set_mask(unsigned long mask, unsigned long *addr) +static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) { -#ifdef CONFIG_SMP - int status; - - asm volatile( - "1: mov %3,(_AAR,%2) \n" - " mov (_ADR,%2),%0 \n" - " or %4,%0 \n" - " mov %0,(_ADR,%2) \n" - " mov (_ADR,%2),%0 \n" /* flush */ - " mov (_ASR,%2),%0 \n" - " or %0,%0 \n" - " bne 1b \n" - : "=&r"(status), "=m"(*addr) - : "a"(ATOMIC_OPS_BASE_ADDR), "r"(addr), "r"(mask) - : "memory", "cc"); -#else - unsigned long flags; - - flags = arch_local_cli_save(); - *addr |= mask; - arch_local_irq_restore(flags); -#endif + atomic_or(mask, v); } #endif /* __KERNEL__ */ -- cgit v1.2.3 From aebea9359623b58b54242e5daa24e09d6104977b Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 19:47:25 +0200 Subject: parisc: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/parisc/include/asm/atomic.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'arch') diff --git a/arch/parisc/include/asm/atomic.h b/arch/parisc/include/asm/atomic.h index 226f8ca993f6..be2c50ddebd6 100644 --- a/arch/parisc/include/asm/atomic.h +++ b/arch/parisc/include/asm/atomic.h @@ -126,6 +126,12 @@ static __inline__ int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add, +=) ATOMIC_OPS(sub, -=) +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC_OP(and, &=) +ATOMIC_OP(or, |=) +ATOMIC_OP(xor, ^=) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -185,6 +191,9 @@ static __inline__ s64 atomic64_##op##_return(s64 i, atomic64_t *v) \ ATOMIC64_OPS(add, +=) ATOMIC64_OPS(sub, -=) +ATOMIC64_OP(and, &=) +ATOMIC64_OP(or, |=) +ATOMIC64_OP(xor, ^=) #undef ATOMIC64_OPS #undef ATOMIC64_OP_RETURN -- cgit v1.2.3 From d0b7eb6ff2985c9350fdb297fa5727c35324d4df Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 19:46:23 +0200 Subject: powerpc: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Acked-by: Benjamin Herrenschmidt Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/powerpc/include/asm/atomic.h | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'arch') diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h index 512d2782b043..6ca89e2aca15 100644 --- a/arch/powerpc/include/asm/atomic.h +++ b/arch/powerpc/include/asm/atomic.h @@ -67,6 +67,12 @@ static __inline__ int atomic_##op##_return(int a, atomic_t *v) \ ATOMIC_OPS(add, add) ATOMIC_OPS(sub, subf) +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC_OP(and, and) +ATOMIC_OP(or, or) +ATOMIC_OP(xor, xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -304,6 +310,9 @@ static __inline__ long atomic64_##op##_return(long a, atomic64_t *v) \ ATOMIC64_OPS(add, add) ATOMIC64_OPS(sub, subf) +ATOMIC64_OP(and, and) +ATOMIC64_OP(or, or) +ATOMIC64_OP(xor, xor) #undef ATOMIC64_OPS #undef ATOMIC64_OP_RETURN -- cgit v1.2.3 From 658aa51459c2f5284183d35b6dd0beca0e0bfe2f Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 19:44:20 +0200 Subject: sh: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/sh/include/asm/atomic-grb.h | 45 ++++++--------------------------------- arch/sh/include/asm/atomic-irq.h | 21 +++--------------- arch/sh/include/asm/atomic-llsc.h | 31 +++------------------------ arch/sh/include/asm/atomic.h | 10 +++++++++ 4 files changed, 22 insertions(+), 85 deletions(-) (limited to 'arch') diff --git a/arch/sh/include/asm/atomic-grb.h b/arch/sh/include/asm/atomic-grb.h index 97a5fda83450..4b03830d48c7 100644 --- a/arch/sh/include/asm/atomic-grb.h +++ b/arch/sh/include/asm/atomic-grb.h @@ -48,47 +48,14 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - int tmp; - unsigned int _mask = ~mask; - - __asm__ __volatile__ ( - " .align 2 \n\t" - " mova 1f, r0 \n\t" /* r0 = end point */ - " mov r15, r1 \n\t" /* r1 = saved sp */ - " mov #-6, r15 \n\t" /* LOGIN: r15 = size */ - " mov.l @%1, %0 \n\t" /* load old value */ - " and %2, %0 \n\t" /* add */ - " mov.l %0, @%1 \n\t" /* store new value */ - "1: mov r1, r15 \n\t" /* LOGOUT */ - : "=&r" (tmp), - "+r" (v) - : "r" (_mask) - : "memory" , "r0", "r1"); -} - -static inline void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - int tmp; - - __asm__ __volatile__ ( - " .align 2 \n\t" - " mova 1f, r0 \n\t" /* r0 = end point */ - " mov r15, r1 \n\t" /* r1 = saved sp */ - " mov #-6, r15 \n\t" /* LOGIN: r15 = size */ - " mov.l @%1, %0 \n\t" /* load old value */ - " or %2, %0 \n\t" /* or */ - " mov.l %0, @%1 \n\t" /* store new value */ - "1: mov r1, r15 \n\t" /* LOGOUT */ - : "=&r" (tmp), - "+r" (v) - : "r" (mask) - : "memory" , "r0", "r1"); -} - #endif /* __ASM_SH_ATOMIC_GRB_H */ diff --git a/arch/sh/include/asm/atomic-irq.h b/arch/sh/include/asm/atomic-irq.h index 61d107523f06..23fcdad5773e 100644 --- a/arch/sh/include/asm/atomic-irq.h +++ b/arch/sh/include/asm/atomic-irq.h @@ -37,27 +37,12 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add, +=) ATOMIC_OPS(sub, -=) +ATOMIC_OP(and, &=) +ATOMIC_OP(or, |=) +ATOMIC_OP(xor, ^=) #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - unsigned long flags; - - raw_local_irq_save(flags); - v->counter &= ~mask; - raw_local_irq_restore(flags); -} - -static inline void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - unsigned long flags; - - raw_local_irq_save(flags); - v->counter |= mask; - raw_local_irq_restore(flags); -} - #endif /* __ASM_SH_ATOMIC_IRQ_H */ diff --git a/arch/sh/include/asm/atomic-llsc.h b/arch/sh/include/asm/atomic-llsc.h index 8575dccb9ef7..33d34b16d4d6 100644 --- a/arch/sh/include/asm/atomic-llsc.h +++ b/arch/sh/include/asm/atomic-llsc.h @@ -52,37 +52,12 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - unsigned long tmp; - - __asm__ __volatile__ ( -"1: movli.l @%2, %0 ! atomic_clear_mask \n" -" and %1, %0 \n" -" movco.l %0, @%2 \n" -" bf 1b \n" - : "=&z" (tmp) - : "r" (~mask), "r" (&v->counter) - : "t"); -} - -static inline void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - unsigned long tmp; - - __asm__ __volatile__ ( -"1: movli.l @%2, %0 ! atomic_set_mask \n" -" or %1, %0 \n" -" movco.l %0, @%2 \n" -" bf 1b \n" - : "=&z" (tmp) - : "r" (mask), "r" (&v->counter) - : "t"); -} - #endif /* __ASM_SH_ATOMIC_LLSC_H */ diff --git a/arch/sh/include/asm/atomic.h b/arch/sh/include/asm/atomic.h index 05b9f74ce2d5..cee0245257e1 100644 --- a/arch/sh/include/asm/atomic.h +++ b/arch/sh/include/asm/atomic.h @@ -25,6 +25,16 @@ #include #endif +static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) +{ + atomic_and(~mask, v); +} + +static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) +{ + atomic_or(mask, v); +} + #define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) #define atomic_dec_return(v) atomic_sub_return(1, (v)) #define atomic_inc_return(v) atomic_add_return(1, (v)) -- cgit v1.2.3 From 304a0d699a3c6103b61d5ea18d56820e7d8e3116 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 19:40:25 +0200 Subject: sparc: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Acked-by: David S. Miller Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/sparc/include/asm/atomic_32.h | 4 ++++ arch/sparc/include/asm/atomic_64.h | 6 ++++++ arch/sparc/lib/atomic32.c | 22 +++++++++++++++++++--- arch/sparc/lib/atomic_64.S | 6 ++++++ arch/sparc/lib/ksyms.c | 3 +++ 5 files changed, 38 insertions(+), 3 deletions(-) (limited to 'arch') diff --git a/arch/sparc/include/asm/atomic_32.h b/arch/sparc/include/asm/atomic_32.h index 0e69b7e7a439..e19d8880b146 100644 --- a/arch/sparc/include/asm/atomic_32.h +++ b/arch/sparc/include/asm/atomic_32.h @@ -17,10 +17,14 @@ #include #include +#define CONFIG_ARCH_HAS_ATOMIC_OR #define ATOMIC_INIT(i) { (i) } int atomic_add_return(int, atomic_t *); +void atomic_and(int, atomic_t *); +void atomic_or(int, atomic_t *); +void atomic_xor(int, atomic_t *); int atomic_cmpxchg(atomic_t *, int, int); int atomic_xchg(atomic_t *, int); int __atomic_add_unless(atomic_t *, int, int); diff --git a/arch/sparc/include/asm/atomic_64.h b/arch/sparc/include/asm/atomic_64.h index 4082749913ce..d6af27c93450 100644 --- a/arch/sparc/include/asm/atomic_64.h +++ b/arch/sparc/include/asm/atomic_64.h @@ -33,6 +33,12 @@ long atomic64_##op##_return(long, atomic64_t *); ATOMIC_OPS(add) ATOMIC_OPS(sub) +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c index 71cd65ab200c..b9d63c0a7aab 100644 --- a/arch/sparc/lib/atomic32.c +++ b/arch/sparc/lib/atomic32.c @@ -27,22 +27,38 @@ static DEFINE_SPINLOCK(dummy); #endif /* SMP */ -#define ATOMIC_OP(op, cop) \ +#define ATOMIC_OP_RETURN(op, c_op) \ int atomic_##op##_return(int i, atomic_t *v) \ { \ int ret; \ unsigned long flags; \ spin_lock_irqsave(ATOMIC_HASH(v), flags); \ \ - ret = (v->counter cop i); \ + ret = (v->counter c_op i); \ \ spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \ return ret; \ } \ EXPORT_SYMBOL(atomic_##op##_return); -ATOMIC_OP(add, +=) +#define ATOMIC_OP(op, c_op) \ +void atomic_##op(int i, atomic_t *v) \ +{ \ + unsigned long flags; \ + spin_lock_irqsave(ATOMIC_HASH(v), flags); \ + \ + v->counter c_op i; \ + \ + spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \ +} \ +EXPORT_SYMBOL(atomic_##op); + +ATOMIC_OP_RETURN(add, +=) +ATOMIC_OP(and, &=) +ATOMIC_OP(or, |=) +ATOMIC_OP(xor, ^=) +#undef ATOMIC_OP_RETURN #undef ATOMIC_OP int atomic_xchg(atomic_t *v, int new) diff --git a/arch/sparc/lib/atomic_64.S b/arch/sparc/lib/atomic_64.S index 05dac43907d1..d6b0363f345b 100644 --- a/arch/sparc/lib/atomic_64.S +++ b/arch/sparc/lib/atomic_64.S @@ -47,6 +47,9 @@ ENDPROC(atomic_##op##_return); ATOMIC_OPS(add) ATOMIC_OPS(sub) +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN @@ -84,6 +87,9 @@ ENDPROC(atomic64_##op##_return); ATOMIC64_OPS(add) ATOMIC64_OPS(sub) +ATOMIC64_OP(and) +ATOMIC64_OP(or) +ATOMIC64_OP(xor) #undef ATOMIC64_OPS #undef ATOMIC64_OP_RETURN diff --git a/arch/sparc/lib/ksyms.c b/arch/sparc/lib/ksyms.c index 1d649a95660c..bb6005997268 100644 --- a/arch/sparc/lib/ksyms.c +++ b/arch/sparc/lib/ksyms.c @@ -111,6 +111,9 @@ EXPORT_SYMBOL(atomic64_##op##_return); ATOMIC_OPS(add) ATOMIC_OPS(sub) +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN -- cgit v1.2.3 From 2a3ed90f428cab65567c5421ebc0f6c8d02c1216 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 19:35:00 +0200 Subject: xtensa: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/xtensa/include/asm/atomic.h | 85 ++++++++-------------------------------- 1 file changed, 16 insertions(+), 69 deletions(-) (limited to 'arch') diff --git a/arch/xtensa/include/asm/atomic.h b/arch/xtensa/include/asm/atomic.h index 00b7d46b35b8..4dd2450300a6 100644 --- a/arch/xtensa/include/asm/atomic.h +++ b/arch/xtensa/include/asm/atomic.h @@ -145,10 +145,26 @@ static inline int atomic_##op##_return(int i, atomic_t * v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP +static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) +{ + atomic_or(mask, v); +} + +static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) +{ + atomic_and(~mask, v); +} + /** * atomic_sub_and_test - subtract value from variable and test result * @i: integer value to subtract @@ -250,75 +266,6 @@ static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) return c; } - -static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ -#if XCHAL_HAVE_S32C1I - unsigned long tmp; - int result; - - __asm__ __volatile__( - "1: l32i %1, %3, 0\n" - " wsr %1, scompare1\n" - " and %0, %1, %2\n" - " s32c1i %0, %3, 0\n" - " bne %0, %1, 1b\n" - : "=&a" (result), "=&a" (tmp) - : "a" (~mask), "a" (v) - : "memory" - ); -#else - unsigned int all_f = -1; - unsigned int vval; - - __asm__ __volatile__( - " rsil a15,"__stringify(LOCKLEVEL)"\n" - " l32i %0, %2, 0\n" - " xor %1, %4, %3\n" - " and %0, %0, %4\n" - " s32i %0, %2, 0\n" - " wsr a15, ps\n" - " rsync\n" - : "=&a" (vval), "=a" (mask) - : "a" (v), "a" (all_f), "1" (mask) - : "a15", "memory" - ); -#endif -} - -static inline void atomic_set_mask(unsigned int mask, atomic_t *v) -{ -#if XCHAL_HAVE_S32C1I - unsigned long tmp; - int result; - - __asm__ __volatile__( - "1: l32i %1, %3, 0\n" - " wsr %1, scompare1\n" - " or %0, %1, %2\n" - " s32c1i %0, %3, 0\n" - " bne %0, %1, 1b\n" - : "=&a" (result), "=&a" (tmp) - : "a" (mask), "a" (v) - : "memory" - ); -#else - unsigned int vval; - - __asm__ __volatile__( - " rsil a15,"__stringify(LOCKLEVEL)"\n" - " l32i %0, %2, 0\n" - " or %0, %0, %1\n" - " s32i %0, %2, 0\n" - " wsr a15, ps\n" - " rsync\n" - : "=&a" (vval) - : "a" (mask), "a" (v) - : "a15", "memory" - ); -#endif -} - #endif /* __KERNEL__ */ #endif /* _XTENSA_ATOMIC_H */ -- cgit v1.2.3 From ae8c35c85be92b79c545c57c2a14c2f8136d3353 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 20:15:48 +0200 Subject: s390: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Acked-by: Heiko Carstens Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/s390/include/asm/atomic.h | 47 +++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 14 deletions(-) (limited to 'arch') diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h index adbe3802e377..b3859d8e001f 100644 --- a/arch/s390/include/asm/atomic.h +++ b/arch/s390/include/asm/atomic.h @@ -27,6 +27,7 @@ #define __ATOMIC_OR "lao" #define __ATOMIC_AND "lan" #define __ATOMIC_ADD "laa" +#define __ATOMIC_XOR "lax" #define __ATOMIC_BARRIER "bcr 14,0\n" #define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier) \ @@ -49,6 +50,7 @@ #define __ATOMIC_OR "or" #define __ATOMIC_AND "nr" #define __ATOMIC_ADD "ar" +#define __ATOMIC_XOR "xr" #define __ATOMIC_BARRIER "\n" #define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier) \ @@ -118,14 +120,26 @@ static inline void atomic_add(int i, atomic_t *v) #define atomic_dec_return(_v) atomic_sub_return(1, _v) #define atomic_dec_and_test(_v) (atomic_sub_return(1, _v) == 0) -static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) +#define ATOMIC_OP(op, OP) \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + __ATOMIC_LOOP(v, i, __ATOMIC_##OP, __ATOMIC_NO_BARRIER); \ +} + +ATOMIC_OP(and, AND) +ATOMIC_OP(or, OR) +ATOMIC_OP(xor, XOR) + +#undef ATOMIC_OP + +static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) { - __ATOMIC_LOOP(v, ~mask, __ATOMIC_AND, __ATOMIC_NO_BARRIER); + atomic_and(~mask, v); } -static inline void atomic_set_mask(unsigned int mask, atomic_t *v) +static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) { - __ATOMIC_LOOP(v, mask, __ATOMIC_OR, __ATOMIC_NO_BARRIER); + atomic_or(mask, v); } #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) @@ -167,6 +181,7 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) #define __ATOMIC64_OR "laog" #define __ATOMIC64_AND "lang" #define __ATOMIC64_ADD "laag" +#define __ATOMIC64_XOR "laxg" #define __ATOMIC64_BARRIER "bcr 14,0\n" #define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier) \ @@ -189,6 +204,7 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) #define __ATOMIC64_OR "ogr" #define __ATOMIC64_AND "ngr" #define __ATOMIC64_ADD "agr" +#define __ATOMIC64_XOR "xgr" #define __ATOMIC64_BARRIER "\n" #define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier) \ @@ -247,16 +263,6 @@ static inline void atomic64_add(long long i, atomic64_t *v) __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_NO_BARRIER); } -static inline void atomic64_clear_mask(unsigned long mask, atomic64_t *v) -{ - __ATOMIC64_LOOP(v, ~mask, __ATOMIC64_AND, __ATOMIC64_NO_BARRIER); -} - -static inline void atomic64_set_mask(unsigned long mask, atomic64_t *v) -{ - __ATOMIC64_LOOP(v, mask, __ATOMIC64_OR, __ATOMIC64_NO_BARRIER); -} - #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) static inline long long atomic64_cmpxchg(atomic64_t *v, @@ -270,6 +276,19 @@ static inline long long atomic64_cmpxchg(atomic64_t *v, return old; } +#define ATOMIC64_OP(op, OP) \ +static inline void atomic64_##op(long i, atomic64_t *v) \ +{ \ + __ATOMIC64_LOOP(v, i, __ATOMIC64_##OP, __ATOMIC64_NO_BARRIER); \ +} + +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC64_OP(and, AND) +ATOMIC64_OP(or, OR) +ATOMIC64_OP(xor, XOR) + +#undef ATOMIC64_OP #undef __ATOMIC64_LOOP static inline int atomic64_add_unless(atomic64_t *v, long long i, long long u) -- cgit v1.2.3 From 7fc1845dd45a825b3c2b760df342a94f61fb1113 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 20:28:37 +0200 Subject: x86: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/x86/include/asm/atomic.h | 33 +++++++++++++++++++++++++-------- arch/x86/include/asm/atomic64_32.h | 14 ++++++++++++++ arch/x86/include/asm/atomic64_64.h | 15 +++++++++++++++ 3 files changed, 54 insertions(+), 8 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h index e9168955c42f..f3a3ec040694 100644 --- a/arch/x86/include/asm/atomic.h +++ b/arch/x86/include/asm/atomic.h @@ -182,6 +182,23 @@ static inline int atomic_xchg(atomic_t *v, int new) return xchg(&v->counter, new); } +#define ATOMIC_OP(op) \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + asm volatile(LOCK_PREFIX #op"l %1,%0" \ + : "+m" (v->counter) \ + : "ir" (i) \ + : "memory"); \ +} + +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + +#undef ATOMIC_OP + /** * __atomic_add_unless - add unless the number is already a given value * @v: pointer of type atomic_t @@ -219,15 +236,15 @@ static __always_inline short int atomic_inc_short(short int *v) return *v; } -/* These are x86-specific, used by some header files */ -#define atomic_clear_mask(mask, addr) \ - asm volatile(LOCK_PREFIX "andl %0,%1" \ - : : "r" (~(mask)), "m" (*(addr)) : "memory") +static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) +{ + atomic_and(~mask, v); +} -#define atomic_set_mask(mask, addr) \ - asm volatile(LOCK_PREFIX "orl %0,%1" \ - : : "r" ((unsigned)(mask)), "m" (*(addr)) \ - : "memory") +static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) +{ + atomic_or(mask, v); +} #ifdef CONFIG_X86_32 # include diff --git a/arch/x86/include/asm/atomic64_32.h b/arch/x86/include/asm/atomic64_32.h index b154de75c90c..a11c30b77fb5 100644 --- a/arch/x86/include/asm/atomic64_32.h +++ b/arch/x86/include/asm/atomic64_32.h @@ -313,4 +313,18 @@ static inline long long atomic64_dec_if_positive(atomic64_t *v) #undef alternative_atomic64 #undef __alternative_atomic64 +#define ATOMIC64_OP(op, c_op) \ +static inline void atomic64_##op(long long i, atomic64_t *v) \ +{ \ + long long old, c = 0; \ + while ((old = atomic64_cmpxchg(v, c, c c_op i)) != c) \ + c = old; \ +} + +ATOMIC64_OP(and, &) +ATOMIC64_OP(or, |) +ATOMIC64_OP(xor, ^) + +#undef ATOMIC64_OP + #endif /* _ASM_X86_ATOMIC64_32_H */ diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atomic64_64.h index b965f9e03f2a..50e33eff58de 100644 --- a/arch/x86/include/asm/atomic64_64.h +++ b/arch/x86/include/asm/atomic64_64.h @@ -220,4 +220,19 @@ static inline long atomic64_dec_if_positive(atomic64_t *v) return dec; } +#define ATOMIC64_OP(op) \ +static inline void atomic64_##op(long i, atomic64_t *v) \ +{ \ + asm volatile(LOCK_PREFIX #op"q %1,%0" \ + : "+m" (v->counter) \ + : "er" (i) \ + : "memory"); \ +} + +ATOMIC64_OP(and) +ATOMIC64_OP(or) +ATOMIC64_OP(xor) + +#undef ATOMIC64_OP + #endif /* _ASM_X86_ATOMIC64_64_H */ -- cgit v1.2.3 From b0d8003ef405c4148b703cdaab1171045c6c3bbd Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 24 Apr 2015 00:49:20 +0200 Subject: frv: Rewrite atomic implementation Mostly complete rewrite of the FRV atomic implementation, instead of using assembly files, use inline assembler. The out-of-line CONFIG option makes a bit of a mess of things, but a little CPP trickery gets that done too. FRV already had the atomic logic ops but under a non standard name, the reimplementation provides the generic names and provides the intermediate form required for the bitops implementation. The slightly inconsistent __atomic32_fetch_##op naming is because __atomic_fetch_##op conlicts with GCC builtin functions. The 64bit atomic ops use the inline assembly %Ln construct to access the low word register (r+1), afaik this construct was not previously used in the kernel and is completely undocumented, but I found it in the FRV GCC code and it seems to work. FRV had a non-standard definition of atomic_{clear,set}_mask() which would work types other than atomic_t, the one user relying on that (arch/frv/kernel/dma.c) got converted to use the new intermediate form. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/frv/include/asm/atomic.h | 119 +++++++++++++------------ arch/frv/include/asm/atomic_defs.h | 172 +++++++++++++++++++++++++++++++++++++ arch/frv/include/asm/bitops.h | 99 +++------------------ arch/frv/kernel/dma.c | 6 +- arch/frv/kernel/frv_ksyms.c | 5 -- arch/frv/lib/Makefile | 2 +- arch/frv/lib/atomic-lib.c | 7 ++ arch/frv/lib/atomic-ops.S | 110 ------------------------ arch/frv/lib/atomic64-ops.S | 94 -------------------- 9 files changed, 259 insertions(+), 355 deletions(-) create mode 100644 arch/frv/include/asm/atomic_defs.h create mode 100644 arch/frv/lib/atomic-lib.c (limited to 'arch') diff --git a/arch/frv/include/asm/atomic.h b/arch/frv/include/asm/atomic.h index 102190a61d65..74d22454d7c6 100644 --- a/arch/frv/include/asm/atomic.h +++ b/arch/frv/include/asm/atomic.h @@ -15,7 +15,6 @@ #define _ASM_ATOMIC_H #include -#include #include #include @@ -23,6 +22,8 @@ #error not SMP safe #endif +#include + /* * Atomic operations that C can't guarantee us. Useful for * resource counting etc.. @@ -34,56 +35,26 @@ #define atomic_read(v) ACCESS_ONCE((v)->counter) #define atomic_set(v, i) (((v)->counter) = (i)) -#ifndef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS -static inline int atomic_add_return(int i, atomic_t *v) +static inline int atomic_inc_return(atomic_t *v) { - unsigned long val; + return __atomic_add_return(1, &v->counter); +} - asm("0: \n" - " orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */ - " ckeq icc3,cc7 \n" - " ld.p %M0,%1 \n" /* LD.P/ORCR must be atomic */ - " orcr cc7,cc7,cc3 \n" /* set CC3 to true */ - " add%I2 %1,%2,%1 \n" - " cst.p %1,%M0 ,cc3,#1 \n" - " corcc gr29,gr29,gr0 ,cc3,#1 \n" /* clear ICC3.Z if store happens */ - " beq icc3,#0,0b \n" - : "+U"(v->counter), "=&r"(val) - : "NPr"(i) - : "memory", "cc7", "cc3", "icc3" - ); +static inline int atomic_dec_return(atomic_t *v) +{ + return __atomic_sub_return(1, &v->counter); +} - return val; +static inline int atomic_add_return(int i, atomic_t *v) +{ + return __atomic_add_return(i, &v->counter); } static inline int atomic_sub_return(int i, atomic_t *v) { - unsigned long val; - - asm("0: \n" - " orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */ - " ckeq icc3,cc7 \n" - " ld.p %M0,%1 \n" /* LD.P/ORCR must be atomic */ - " orcr cc7,cc7,cc3 \n" /* set CC3 to true */ - " sub%I2 %1,%2,%1 \n" - " cst.p %1,%M0 ,cc3,#1 \n" - " corcc gr29,gr29,gr0 ,cc3,#1 \n" /* clear ICC3.Z if store happens */ - " beq icc3,#0,0b \n" - : "+U"(v->counter), "=&r"(val) - : "NPr"(i) - : "memory", "cc7", "cc3", "icc3" - ); - - return val; + return __atomic_sub_return(i, &v->counter); } -#else - -extern int atomic_add_return(int i, atomic_t *v); -extern int atomic_sub_return(int i, atomic_t *v); - -#endif - static inline int atomic_add_negative(int i, atomic_t *v) { return atomic_add_return(i, v) < 0; @@ -101,17 +72,14 @@ static inline void atomic_sub(int i, atomic_t *v) static inline void atomic_inc(atomic_t *v) { - atomic_add_return(1, v); + atomic_inc_return(v); } static inline void atomic_dec(atomic_t *v) { - atomic_sub_return(1, v); + atomic_dec_return(v); } -#define atomic_dec_return(v) atomic_sub_return(1, (v)) -#define atomic_inc_return(v) atomic_add_return(1, (v)) - #define atomic_sub_and_test(i,v) (atomic_sub_return((i), (v)) == 0) #define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0) #define atomic_inc_and_test(v) (atomic_add_return(1, (v)) == 0) @@ -120,18 +88,19 @@ static inline void atomic_dec(atomic_t *v) * 64-bit atomic ops */ typedef struct { - volatile long long counter; + long long counter; } atomic64_t; #define ATOMIC64_INIT(i) { (i) } -static inline long long atomic64_read(atomic64_t *v) +static inline long long atomic64_read(const atomic64_t *v) { long long counter; asm("ldd%I1 %M1,%0" : "=e"(counter) : "m"(v->counter)); + return counter; } @@ -142,10 +111,25 @@ static inline void atomic64_set(atomic64_t *v, long long i) : "e"(i)); } -extern long long atomic64_inc_return(atomic64_t *v); -extern long long atomic64_dec_return(atomic64_t *v); -extern long long atomic64_add_return(long long i, atomic64_t *v); -extern long long atomic64_sub_return(long long i, atomic64_t *v); +static inline long long atomic64_inc_return(atomic64_t *v) +{ + return __atomic64_add_return(1, &v->counter); +} + +static inline long long atomic64_dec_return(atomic64_t *v) +{ + return __atomic64_sub_return(1, &v->counter); +} + +static inline long long atomic64_add_return(long long i, atomic64_t *v) +{ + return __atomic64_add_return(i, &v->counter); +} + +static inline long long atomic64_sub_return(long long i, atomic64_t *v) +{ + return __atomic64_sub_return(i, &v->counter); +} static inline long long atomic64_add_negative(long long i, atomic64_t *v) { @@ -176,6 +160,7 @@ static inline void atomic64_dec(atomic64_t *v) #define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0) #define atomic64_inc_and_test(v) (atomic64_inc_return((v)) == 0) + #define atomic_cmpxchg(v, old, new) (cmpxchg(&(v)->counter, old, new)) #define atomic_xchg(v, new) (xchg(&(v)->counter, new)) #define atomic64_cmpxchg(v, old, new) (__cmpxchg_64(old, new, &(v)->counter)) @@ -196,5 +181,33 @@ static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) return c; } +#define ATOMIC_OP(op) \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + (void)__atomic32_fetch_##op(i, &v->counter); \ +} \ + \ +static inline void atomic64_##op(long long i, atomic64_t *v) \ +{ \ + (void)__atomic64_fetch_##op(i, &v->counter); \ +} + +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC_OP(or) +ATOMIC_OP(and) +ATOMIC_OP(xor) + +#undef ATOMIC_OP + +static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) +{ + atomic_and(~mask, v); +} + +static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) +{ + atomic_or(mask, v); +} #endif /* _ASM_ATOMIC_H */ diff --git a/arch/frv/include/asm/atomic_defs.h b/arch/frv/include/asm/atomic_defs.h new file mode 100644 index 000000000000..36e126d2f801 --- /dev/null +++ b/arch/frv/include/asm/atomic_defs.h @@ -0,0 +1,172 @@ + +#include + +#ifdef __ATOMIC_LIB__ + +#ifdef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS + +#define ATOMIC_QUALS +#define ATOMIC_EXPORT(x) EXPORT_SYMBOL(x) + +#else /* !OUTOFLINE && LIB */ + +#define ATOMIC_OP_RETURN(op) +#define ATOMIC_FETCH_OP(op) + +#endif /* OUTOFLINE */ + +#else /* !__ATOMIC_LIB__ */ + +#define ATOMIC_EXPORT(x) + +#ifdef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS + +#define ATOMIC_OP_RETURN(op) \ +extern int __atomic_##op##_return(int i, int *v); \ +extern long long __atomic64_##op##_return(long long i, long long *v); + +#define ATOMIC_FETCH_OP(op) \ +extern int __atomic32_fetch_##op(int i, int *v); \ +extern long long __atomic64_fetch_##op(long long i, long long *v); + +#else /* !OUTOFLINE && !LIB */ + +#define ATOMIC_QUALS static inline + +#endif /* OUTOFLINE */ +#endif /* __ATOMIC_LIB__ */ + + +/* + * Note on the 64 bit inline asm variants... + * + * CSTD is a conditional instruction and needs a constrained memory reference. + * Normally 'U' provides the correct constraints for conditional instructions + * and this is used for the 32 bit version, however 'U' does not appear to work + * for 64 bit values (gcc-4.9) + * + * The exact constraint is that conditional instructions cannot deal with an + * immediate displacement in the memory reference, so what we do is we read the + * address through a volatile cast into a local variable in order to insure we + * _have_ to compute the correct address without displacement. This allows us + * to use the regular 'm' for the memory address. + * + * Furthermore, the %Ln operand, which prints the low word register (r+1), + * really only works for registers, this means we cannot allow immediate values + * for the 64 bit versions -- like we do for the 32 bit ones. + * + */ + +#ifndef ATOMIC_OP_RETURN +#define ATOMIC_OP_RETURN(op) \ +ATOMIC_QUALS int __atomic_##op##_return(int i, int *v) \ +{ \ + int val; \ + \ + asm volatile( \ + "0: \n" \ + " orcc gr0,gr0,gr0,icc3 \n" \ + " ckeq icc3,cc7 \n" \ + " ld.p %M0,%1 \n" \ + " orcr cc7,cc7,cc3 \n" \ + " "#op"%I2 %1,%2,%1 \n" \ + " cst.p %1,%M0 ,cc3,#1 \n" \ + " corcc gr29,gr29,gr0 ,cc3,#1 \n" \ + " beq icc3,#0,0b \n" \ + : "+U"(*v), "=&r"(val) \ + : "NPr"(i) \ + : "memory", "cc7", "cc3", "icc3" \ + ); \ + \ + return val; \ +} \ +ATOMIC_EXPORT(__atomic_##op##_return); \ + \ +ATOMIC_QUALS long long __atomic64_##op##_return(long long i, long long *v) \ +{ \ + long long *__v = READ_ONCE(v); \ + long long val; \ + \ + asm volatile( \ + "0: \n" \ + " orcc gr0,gr0,gr0,icc3 \n" \ + " ckeq icc3,cc7 \n" \ + " ldd.p %M0,%1 \n" \ + " orcr cc7,cc7,cc3 \n" \ + " "#op"cc %L1,%L2,%L1,icc0 \n" \ + " "#op"x %1,%2,%1,icc0 \n" \ + " cstd.p %1,%M0 ,cc3,#1 \n" \ + " corcc gr29,gr29,gr0 ,cc3,#1 \n" \ + " beq icc3,#0,0b \n" \ + : "+m"(*__v), "=&e"(val) \ + : "e"(i) \ + : "memory", "cc7", "cc3", "icc0", "icc3" \ + ); \ + \ + return val; \ +} \ +ATOMIC_EXPORT(__atomic64_##op##_return); +#endif + +#ifndef ATOMIC_FETCH_OP +#define ATOMIC_FETCH_OP(op) \ +ATOMIC_QUALS int __atomic32_fetch_##op(int i, int *v) \ +{ \ + int old, tmp; \ + \ + asm volatile( \ + "0: \n" \ + " orcc gr0,gr0,gr0,icc3 \n" \ + " ckeq icc3,cc7 \n" \ + " ld.p %M0,%1 \n" \ + " orcr cc7,cc7,cc3 \n" \ + " "#op"%I3 %1,%3,%2 \n" \ + " cst.p %2,%M0 ,cc3,#1 \n" \ + " corcc gr29,gr29,gr0 ,cc3,#1 \n" \ + " beq icc3,#0,0b \n" \ + : "+U"(*v), "=&r"(old), "=r"(tmp) \ + : "NPr"(i) \ + : "memory", "cc7", "cc3", "icc3" \ + ); \ + \ + return old; \ +} \ +ATOMIC_EXPORT(__atomic32_fetch_##op); \ + \ +ATOMIC_QUALS long long __atomic64_fetch_##op(long long i, long long *v) \ +{ \ + long long *__v = READ_ONCE(v); \ + long long old, tmp; \ + \ + asm volatile( \ + "0: \n" \ + " orcc gr0,gr0,gr0,icc3 \n" \ + " ckeq icc3,cc7 \n" \ + " ldd.p %M0,%1 \n" \ + " orcr cc7,cc7,cc3 \n" \ + " "#op" %L1,%L3,%L2 \n" \ + " "#op" %1,%3,%2 \n" \ + " cstd.p %2,%M0 ,cc3,#1 \n" \ + " corcc gr29,gr29,gr0 ,cc3,#1 \n" \ + " beq icc3,#0,0b \n" \ + : "+m"(*__v), "=&e"(old), "=e"(tmp) \ + : "e"(i) \ + : "memory", "cc7", "cc3", "icc3" \ + ); \ + \ + return old; \ +} \ +ATOMIC_EXPORT(__atomic64_fetch_##op); +#endif + +ATOMIC_FETCH_OP(or) +ATOMIC_FETCH_OP(and) +ATOMIC_FETCH_OP(xor) + +ATOMIC_OP_RETURN(add) +ATOMIC_OP_RETURN(sub) + +#undef ATOMIC_FETCH_OP +#undef ATOMIC_OP_RETURN +#undef ATOMIC_QUALS +#undef ATOMIC_EXPORT diff --git a/arch/frv/include/asm/bitops.h b/arch/frv/include/asm/bitops.h index 96de220ef131..0df8e95e3715 100644 --- a/arch/frv/include/asm/bitops.h +++ b/arch/frv/include/asm/bitops.h @@ -25,109 +25,30 @@ #include -#ifndef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS -static inline -unsigned long atomic_test_and_ANDNOT_mask(unsigned long mask, volatile unsigned long *v) -{ - unsigned long old, tmp; - - asm volatile( - "0: \n" - " orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */ - " ckeq icc3,cc7 \n" - " ld.p %M0,%1 \n" /* LD.P/ORCR are atomic */ - " orcr cc7,cc7,cc3 \n" /* set CC3 to true */ - " and%I3 %1,%3,%2 \n" - " cst.p %2,%M0 ,cc3,#1 \n" /* if store happens... */ - " corcc gr29,gr29,gr0 ,cc3,#1 \n" /* ... clear ICC3.Z */ - " beq icc3,#0,0b \n" - : "+U"(*v), "=&r"(old), "=r"(tmp) - : "NPr"(~mask) - : "memory", "cc7", "cc3", "icc3" - ); - - return old; -} - -static inline -unsigned long atomic_test_and_OR_mask(unsigned long mask, volatile unsigned long *v) -{ - unsigned long old, tmp; - - asm volatile( - "0: \n" - " orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */ - " ckeq icc3,cc7 \n" - " ld.p %M0,%1 \n" /* LD.P/ORCR are atomic */ - " orcr cc7,cc7,cc3 \n" /* set CC3 to true */ - " or%I3 %1,%3,%2 \n" - " cst.p %2,%M0 ,cc3,#1 \n" /* if store happens... */ - " corcc gr29,gr29,gr0 ,cc3,#1 \n" /* ... clear ICC3.Z */ - " beq icc3,#0,0b \n" - : "+U"(*v), "=&r"(old), "=r"(tmp) - : "NPr"(mask) - : "memory", "cc7", "cc3", "icc3" - ); - - return old; -} - -static inline -unsigned long atomic_test_and_XOR_mask(unsigned long mask, volatile unsigned long *v) -{ - unsigned long old, tmp; - - asm volatile( - "0: \n" - " orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */ - " ckeq icc3,cc7 \n" - " ld.p %M0,%1 \n" /* LD.P/ORCR are atomic */ - " orcr cc7,cc7,cc3 \n" /* set CC3 to true */ - " xor%I3 %1,%3,%2 \n" - " cst.p %2,%M0 ,cc3,#1 \n" /* if store happens... */ - " corcc gr29,gr29,gr0 ,cc3,#1 \n" /* ... clear ICC3.Z */ - " beq icc3,#0,0b \n" - : "+U"(*v), "=&r"(old), "=r"(tmp) - : "NPr"(mask) - : "memory", "cc7", "cc3", "icc3" - ); - - return old; -} - -#else - -extern unsigned long atomic_test_and_ANDNOT_mask(unsigned long mask, volatile unsigned long *v); -extern unsigned long atomic_test_and_OR_mask(unsigned long mask, volatile unsigned long *v); -extern unsigned long atomic_test_and_XOR_mask(unsigned long mask, volatile unsigned long *v); - -#endif - -#define atomic_clear_mask(mask, v) atomic_test_and_ANDNOT_mask((mask), (v)) -#define atomic_set_mask(mask, v) atomic_test_and_OR_mask((mask), (v)) +#include static inline int test_and_clear_bit(unsigned long nr, volatile void *addr) { - volatile unsigned long *ptr = addr; - unsigned long mask = 1UL << (nr & 31); + unsigned int *ptr = (void *)addr; + unsigned int mask = 1UL << (nr & 31); ptr += nr >> 5; - return (atomic_test_and_ANDNOT_mask(mask, ptr) & mask) != 0; + return (__atomic32_fetch_and(~mask, ptr) & mask) != 0; } static inline int test_and_set_bit(unsigned long nr, volatile void *addr) { - volatile unsigned long *ptr = addr; - unsigned long mask = 1UL << (nr & 31); + unsigned int *ptr = (void *)addr; + unsigned int mask = 1UL << (nr & 31); ptr += nr >> 5; - return (atomic_test_and_OR_mask(mask, ptr) & mask) != 0; + return (__atomic32_fetch_or(mask, ptr) & mask) != 0; } static inline int test_and_change_bit(unsigned long nr, volatile void *addr) { - volatile unsigned long *ptr = addr; - unsigned long mask = 1UL << (nr & 31); + unsigned int *ptr = (void *)addr; + unsigned int mask = 1UL << (nr & 31); ptr += nr >> 5; - return (atomic_test_and_XOR_mask(mask, ptr) & mask) != 0; + return (__atomic32_fetch_xor(mask, ptr) & mask) != 0; } static inline void clear_bit(unsigned long nr, volatile void *addr) diff --git a/arch/frv/kernel/dma.c b/arch/frv/kernel/dma.c index 156184e17e57..370dc9fa0b11 100644 --- a/arch/frv/kernel/dma.c +++ b/arch/frv/kernel/dma.c @@ -109,13 +109,13 @@ static struct frv_dma_channel frv_dma_channels[FRV_DMA_NCHANS] = { static DEFINE_RWLOCK(frv_dma_channels_lock); -unsigned long frv_dma_inprogress; +unsigned int frv_dma_inprogress; #define frv_clear_dma_inprogress(channel) \ - atomic_clear_mask(1 << (channel), &frv_dma_inprogress); + (void)__atomic32_fetch_and(~(1 << (channel)), &frv_dma_inprogress); #define frv_set_dma_inprogress(channel) \ - atomic_set_mask(1 << (channel), &frv_dma_inprogress); + (void)__atomic32_fetch_or(1 << (channel), &frv_dma_inprogress); /*****************************************************************************/ /* diff --git a/arch/frv/kernel/frv_ksyms.c b/arch/frv/kernel/frv_ksyms.c index 86c516d96dcd..cdb4ce9960eb 100644 --- a/arch/frv/kernel/frv_ksyms.c +++ b/arch/frv/kernel/frv_ksyms.c @@ -58,11 +58,6 @@ EXPORT_SYMBOL(__outsl_ns); EXPORT_SYMBOL(__insl_ns); #ifdef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS -EXPORT_SYMBOL(atomic_test_and_ANDNOT_mask); -EXPORT_SYMBOL(atomic_test_and_OR_mask); -EXPORT_SYMBOL(atomic_test_and_XOR_mask); -EXPORT_SYMBOL(atomic_add_return); -EXPORT_SYMBOL(atomic_sub_return); EXPORT_SYMBOL(__xchg_32); EXPORT_SYMBOL(__cmpxchg_32); #endif diff --git a/arch/frv/lib/Makefile b/arch/frv/lib/Makefile index 4ff2fb1e6b16..970e8b4f1a02 100644 --- a/arch/frv/lib/Makefile +++ b/arch/frv/lib/Makefile @@ -5,4 +5,4 @@ lib-y := \ __ashldi3.o __lshrdi3.o __muldi3.o __ashrdi3.o __negdi2.o __ucmpdi2.o \ checksum.o memcpy.o memset.o atomic-ops.o atomic64-ops.o \ - outsl_ns.o outsl_sw.o insl_ns.o insl_sw.o cache.o + outsl_ns.o outsl_sw.o insl_ns.o insl_sw.o cache.o atomic-lib.o diff --git a/arch/frv/lib/atomic-lib.c b/arch/frv/lib/atomic-lib.c new file mode 100644 index 000000000000..4d1b887c248b --- /dev/null +++ b/arch/frv/lib/atomic-lib.c @@ -0,0 +1,7 @@ + +#include +#include + +#define __ATOMIC_LIB__ + +#include diff --git a/arch/frv/lib/atomic-ops.S b/arch/frv/lib/atomic-ops.S index 5e9e6ab5dd0e..b7439a960b5b 100644 --- a/arch/frv/lib/atomic-ops.S +++ b/arch/frv/lib/atomic-ops.S @@ -17,116 +17,6 @@ .text .balign 4 -############################################################################### -# -# unsigned long atomic_test_and_ANDNOT_mask(unsigned long mask, volatile unsigned long *v); -# -############################################################################### - .globl atomic_test_and_ANDNOT_mask - .type atomic_test_and_ANDNOT_mask,@function -atomic_test_and_ANDNOT_mask: - not.p gr8,gr10 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ld.p @(gr9,gr0),gr8 /* LD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - and gr8,gr10,gr11 - cst.p gr11,@(gr9,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic_test_and_ANDNOT_mask, .-atomic_test_and_ANDNOT_mask - -############################################################################### -# -# unsigned long atomic_test_and_OR_mask(unsigned long mask, volatile unsigned long *v); -# -############################################################################### - .globl atomic_test_and_OR_mask - .type atomic_test_and_OR_mask,@function -atomic_test_and_OR_mask: - or.p gr8,gr8,gr10 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ld.p @(gr9,gr0),gr8 /* LD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - or gr8,gr10,gr11 - cst.p gr11,@(gr9,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic_test_and_OR_mask, .-atomic_test_and_OR_mask - -############################################################################### -# -# unsigned long atomic_test_and_XOR_mask(unsigned long mask, volatile unsigned long *v); -# -############################################################################### - .globl atomic_test_and_XOR_mask - .type atomic_test_and_XOR_mask,@function -atomic_test_and_XOR_mask: - or.p gr8,gr8,gr10 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ld.p @(gr9,gr0),gr8 /* LD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - xor gr8,gr10,gr11 - cst.p gr11,@(gr9,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic_test_and_XOR_mask, .-atomic_test_and_XOR_mask - -############################################################################### -# -# int atomic_add_return(int i, atomic_t *v) -# -############################################################################### - .globl atomic_add_return - .type atomic_add_return,@function -atomic_add_return: - or.p gr8,gr8,gr10 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ld.p @(gr9,gr0),gr8 /* LD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - add gr8,gr10,gr8 - cst.p gr8,@(gr9,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic_add_return, .-atomic_add_return - -############################################################################### -# -# int atomic_sub_return(int i, atomic_t *v) -# -############################################################################### - .globl atomic_sub_return - .type atomic_sub_return,@function -atomic_sub_return: - or.p gr8,gr8,gr10 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ld.p @(gr9,gr0),gr8 /* LD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - sub gr8,gr10,gr8 - cst.p gr8,@(gr9,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic_sub_return, .-atomic_sub_return - ############################################################################### # # uint32_t __xchg_32(uint32_t i, uint32_t *v) diff --git a/arch/frv/lib/atomic64-ops.S b/arch/frv/lib/atomic64-ops.S index b6194eeac127..c4c472308a33 100644 --- a/arch/frv/lib/atomic64-ops.S +++ b/arch/frv/lib/atomic64-ops.S @@ -18,100 +18,6 @@ .balign 4 -############################################################################### -# -# long long atomic64_inc_return(atomic64_t *v) -# -############################################################################### - .globl atomic64_inc_return - .type atomic64_inc_return,@function -atomic64_inc_return: - or.p gr8,gr8,gr10 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ldd.p @(gr10,gr0),gr8 /* LDD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - addicc gr9,#1,gr9,icc0 - addxi gr8,#0,gr8,icc0 - cstd.p gr8,@(gr10,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic64_inc_return, .-atomic64_inc_return - -############################################################################### -# -# long long atomic64_dec_return(atomic64_t *v) -# -############################################################################### - .globl atomic64_dec_return - .type atomic64_dec_return,@function -atomic64_dec_return: - or.p gr8,gr8,gr10 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ldd.p @(gr10,gr0),gr8 /* LDD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - subicc gr9,#1,gr9,icc0 - subxi gr8,#0,gr8,icc0 - cstd.p gr8,@(gr10,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic64_dec_return, .-atomic64_dec_return - -############################################################################### -# -# long long atomic64_add_return(long long i, atomic64_t *v) -# -############################################################################### - .globl atomic64_add_return - .type atomic64_add_return,@function -atomic64_add_return: - or.p gr8,gr8,gr4 - or gr9,gr9,gr5 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ldd.p @(gr10,gr0),gr8 /* LDD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - addcc gr9,gr5,gr9,icc0 - addx gr8,gr4,gr8,icc0 - cstd.p gr8,@(gr10,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic64_add_return, .-atomic64_add_return - -############################################################################### -# -# long long atomic64_sub_return(long long i, atomic64_t *v) -# -############################################################################### - .globl atomic64_sub_return - .type atomic64_sub_return,@function -atomic64_sub_return: - or.p gr8,gr8,gr4 - or gr9,gr9,gr5 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ldd.p @(gr10,gr0),gr8 /* LDD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - subcc gr9,gr5,gr9,icc0 - subx gr8,gr4,gr8,icc0 - cstd.p gr8,@(gr10,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic64_sub_return, .-atomic64_sub_return - ############################################################################### # # uint64_t __xchg_64(uint64_t i, uint64_t *v) -- cgit v1.2.3 From 73ada3700bbb0a4c7cc06ea8d74e93c689f90cdb Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 10 Jul 2015 12:55:45 +0200 Subject: h8300: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and} These will replace the atomic_{set,clear}_mask functions that are available on some archs. Also rework the atomic implementation in terms of CPP macros to avoid the typical repetition -- I seem to have missed this arch the last time around when I did that. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/h8300/include/asm/atomic.h | 135 ++++++++++++---------------------------- 1 file changed, 40 insertions(+), 95 deletions(-) (limited to 'arch') diff --git a/arch/h8300/include/asm/atomic.h b/arch/h8300/include/asm/atomic.h index 7ca73f8546cc..f181f820be33 100644 --- a/arch/h8300/include/asm/atomic.h +++ b/arch/h8300/include/asm/atomic.h @@ -16,83 +16,54 @@ #include -static inline int atomic_add_return(int i, atomic_t *v) -{ - h8300flags flags; - int ret; - - flags = arch_local_irq_save(); - ret = v->counter += i; - arch_local_irq_restore(flags); - return ret; +#define ATOMIC_OP_RETURN(op, c_op) \ +static inline int atomic_##op##_return(int i, atomic_t *v) \ +{ \ + h8300flags flags; \ + int ret; \ + \ + flags = arch_local_irq_save(); \ + ret = v->counter c_op i; \ + arch_local_irq_restore(flags); \ + return ret; \ } -#define atomic_add(i, v) atomic_add_return(i, v) -#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) - -static inline int atomic_sub_return(int i, atomic_t *v) -{ - h8300flags flags; - int ret; - - flags = arch_local_irq_save(); - ret = v->counter -= i; - arch_local_irq_restore(flags); - return ret; +#define ATOMIC_OP(op, c_op) \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + h8300flags flags; \ + \ + flags = arch_local_irq_save(); \ + v->counter c_op i; \ + arch_local_irq_restore(flags); \ } -#define atomic_sub(i, v) atomic_sub_return(i, v) -#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) +ATOMIC_OP_RETURN(add, +=) +ATOMIC_OP_RETURN(sub, -=) -static inline int atomic_inc_return(atomic_t *v) -{ - h8300flags flags; - int ret; - - flags = arch_local_irq_save(); - v->counter++; - ret = v->counter; - arch_local_irq_restore(flags); - return ret; -} +#define CONFIG_ARCH_HAS_ATOMIC_OR -#define atomic_inc(v) atomic_inc_return(v) +ATOMIC_OP(and, &=) +ATOMIC_OP(or, |=) +ATOMIC_OP(xor, ^=) -/* - * atomic_inc_and_test - increment and test - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) +#undef ATOMIC_OP_RETURN +#undef ATOMIC_OP -static inline int atomic_dec_return(atomic_t *v) -{ - h8300flags flags; - int ret; +#define atomic_add(i, v) (void)atomic_add_return(i, v) +#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) - flags = arch_local_irq_save(); - --v->counter; - ret = v->counter; - arch_local_irq_restore(flags); - return ret; -} +#define atomic_sub(i, v) (void)atomic_sub_return(i, v) +#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) -#define atomic_dec(v) atomic_dec_return(v) +#define atomic_inc_return(v) atomic_add_return(1, v) +#define atomic_dec_return(v) atomic_sub_return(1, v) -static inline int atomic_dec_and_test(atomic_t *v) -{ - h8300flags flags; - int ret; +#define atomic_inc(v) (void)atomic_inc_return(v) +#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) - flags = arch_local_irq_save(); - --v->counter; - ret = v->counter; - arch_local_irq_restore(flags); - return ret == 0; -} +#define atomic_dec(v) (void)atomic_dec_return(v) +#define atomic_dec_and_test(v) (atomic_dec_return(v) == 0) static inline int atomic_cmpxchg(atomic_t *v, int old, int new) { @@ -120,40 +91,14 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) return ret; } -static inline void atomic_clear_mask(unsigned long mask, unsigned long *v) +static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) { - unsigned char ccr; - unsigned long tmp; - - __asm__ __volatile__("stc ccr,%w3\n\t" - "orc #0x80,ccr\n\t" - "mov.l %0,%1\n\t" - "and.l %2,%1\n\t" - "mov.l %1,%0\n\t" - "ldc %w3,ccr" - : "=m"(*v), "=r"(tmp) - : "g"(~(mask)), "r"(ccr)); + atomic_and(~mask, v); } -static inline void atomic_set_mask(unsigned long mask, unsigned long *v) +static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) { - unsigned char ccr; - unsigned long tmp; - - __asm__ __volatile__("stc ccr,%w3\n\t" - "orc #0x80,ccr\n\t" - "mov.l %0,%1\n\t" - "or.l %2,%1\n\t" - "mov.l %1,%0\n\t" - "ldc %w3,ccr" - : "=m"(*v), "=r"(tmp) - : "g"(~(mask)), "r"(ccr)); + atomic_or(mask, v); } -/* Atomic operations are already serializing */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - #endif /* __ARCH_H8300_ATOMIC __ */ -- cgit v1.2.3 From 2957c035395e492463d7f589af9dd32388967bbb Mon Sep 17 00:00:00 2001 From: Chris Metcalf Date: Thu, 9 Jul 2015 16:38:17 -0400 Subject: tile: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. For tilegx, these are relatively straightforward; the architecture provides atomic "or" and "and", both 32-bit and 64-bit. To support xor we provide a loop using "cmpexch". For the older 32-bit tilepro architecture, we have to extend the set of low-level assembly routines to include 32-bit "and", as well as all three 64-bit routines. Somewhat confusingly, some 32-bit versions are already used by the bitops inlines, with parameter types appropriate for bitops, so we have to do a bit of casting to match "int" to "unsigned long". Signed-off-by: Chris Metcalf Signed-off-by: Peter Zijlstra (Intel) Link: http://lkml.kernel.org/r/1436474297-32187-1-git-send-email-cmetcalf@ezchip.com Signed-off-by: Thomas Gleixner --- arch/tile/include/asm/atomic_32.h | 30 ++++++++++++++++++++++++++++ arch/tile/include/asm/atomic_64.h | 42 +++++++++++++++++++++++++++++++++++++++ arch/tile/lib/atomic_32.c | 23 +++++++++++++++++++++ arch/tile/lib/atomic_asm_32.S | 4 ++++ 4 files changed, 99 insertions(+) (limited to 'arch') diff --git a/arch/tile/include/asm/atomic_32.h b/arch/tile/include/asm/atomic_32.h index 1b109fad9fff..94237922f0dd 100644 --- a/arch/tile/include/asm/atomic_32.h +++ b/arch/tile/include/asm/atomic_32.h @@ -34,6 +34,21 @@ static inline void atomic_add(int i, atomic_t *v) _atomic_xchg_add(&v->counter, i); } +#define ATOMIC_OP(op) \ +unsigned long _atomic_##op(volatile unsigned long *p, unsigned long mask); \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + _atomic_##op((unsigned long *)&v->counter, i); \ +} + +#define CONFIG_ARCH_HAS_ATOMIC_OR + +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + +#undef ATOMIC_OP + /** * atomic_add_return - add integer and return * @v: pointer of type atomic_t @@ -113,6 +128,17 @@ static inline void atomic64_add(long long i, atomic64_t *v) _atomic64_xchg_add(&v->counter, i); } +#define ATOMIC64_OP(op) \ +long long _atomic64_##op(long long *v, long long n); \ +static inline void atomic64_##op(long long i, atomic64_t *v) \ +{ \ + _atomic64_##op(&v->counter, i); \ +} + +ATOMIC64_OP(and) +ATOMIC64_OP(or) +ATOMIC64_OP(xor) + /** * atomic64_add_return - add integer and return * @v: pointer of type atomic64_t @@ -225,6 +251,7 @@ extern struct __get_user __atomic_xchg_add(volatile int *p, int *lock, int n); extern struct __get_user __atomic_xchg_add_unless(volatile int *p, int *lock, int o, int n); extern struct __get_user __atomic_or(volatile int *p, int *lock, int n); +extern struct __get_user __atomic_and(volatile int *p, int *lock, int n); extern struct __get_user __atomic_andn(volatile int *p, int *lock, int n); extern struct __get_user __atomic_xor(volatile int *p, int *lock, int n); extern long long __atomic64_cmpxchg(volatile long long *p, int *lock, @@ -234,6 +261,9 @@ extern long long __atomic64_xchg_add(volatile long long *p, int *lock, long long n); extern long long __atomic64_xchg_add_unless(volatile long long *p, int *lock, long long o, long long n); +extern long long __atomic64_and(volatile long long *p, int *lock, long long n); +extern long long __atomic64_or(volatile long long *p, int *lock, long long n); +extern long long __atomic64_xor(volatile long long *p, int *lock, long long n); /* Return failure from the atomic wrappers. */ struct __get_user __atomic_bad_address(int __user *addr); diff --git a/arch/tile/include/asm/atomic_64.h b/arch/tile/include/asm/atomic_64.h index 0496970cef82..d07d9fc6e2a1 100644 --- a/arch/tile/include/asm/atomic_64.h +++ b/arch/tile/include/asm/atomic_64.h @@ -58,6 +58,28 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) return oldval; } +#define CONFIG_ARCH_HAS_ATOMIC_OR + +static inline void atomic_and(int i, atomic_t *v) +{ + __insn_fetchand4((void *)&v->counter, i); +} + +static inline void atomic_or(int i, atomic_t *v) +{ + __insn_fetchor4((void *)&v->counter, i); +} + +static inline void atomic_xor(int i, atomic_t *v) +{ + int guess, oldval = v->counter; + do { + guess = oldval; + __insn_mtspr(SPR_CMPEXCH_VALUE, guess); + oldval = __insn_cmpexch4(&v->counter, guess ^ i); + } while (guess != oldval); +} + /* Now the true 64-bit operations. */ #define ATOMIC64_INIT(i) { (i) } @@ -91,6 +113,26 @@ static inline long atomic64_add_unless(atomic64_t *v, long a, long u) return oldval != u; } +static inline void atomic64_and(long i, atomic64_t *v) +{ + __insn_fetchand((void *)&v->counter, i); +} + +static inline void atomic64_or(long i, atomic64_t *v) +{ + __insn_fetchor((void *)&v->counter, i); +} + +static inline void atomic64_xor(long i, atomic64_t *v) +{ + long guess, oldval = v->counter; + do { + guess = oldval; + __insn_mtspr(SPR_CMPEXCH_VALUE, guess); + oldval = __insn_cmpexch(&v->counter, guess ^ i); + } while (guess != oldval); +} + #define atomic64_sub_return(i, v) atomic64_add_return(-(i), (v)) #define atomic64_sub(i, v) atomic64_add(-(i), (v)) #define atomic64_inc_return(v) atomic64_add_return(1, (v)) diff --git a/arch/tile/lib/atomic_32.c b/arch/tile/lib/atomic_32.c index c89b211fd9e7..298df1e9912a 100644 --- a/arch/tile/lib/atomic_32.c +++ b/arch/tile/lib/atomic_32.c @@ -94,6 +94,12 @@ unsigned long _atomic_or(volatile unsigned long *p, unsigned long mask) } EXPORT_SYMBOL(_atomic_or); +unsigned long _atomic_and(volatile unsigned long *p, unsigned long mask) +{ + return __atomic_and((int *)p, __atomic_setup(p), mask).val; +} +EXPORT_SYMBOL(_atomic_and); + unsigned long _atomic_andn(volatile unsigned long *p, unsigned long mask) { return __atomic_andn((int *)p, __atomic_setup(p), mask).val; @@ -136,6 +142,23 @@ long long _atomic64_cmpxchg(long long *v, long long o, long long n) } EXPORT_SYMBOL(_atomic64_cmpxchg); +long long _atomic64_and(long long *v, long long n) +{ + return __atomic64_and(v, __atomic_setup(v), n); +} +EXPORT_SYMBOL(_atomic64_and); + +long long _atomic64_or(long long *v, long long n) +{ + return __atomic64_or(v, __atomic_setup(v), n); +} +EXPORT_SYMBOL(_atomic64_or); + +long long _atomic64_xor(long long *v, long long n) +{ + return __atomic64_xor(v, __atomic_setup(v), n); +} +EXPORT_SYMBOL(_atomic64_xor); /* * If any of the atomic or futex routines hit a bad address (not in diff --git a/arch/tile/lib/atomic_asm_32.S b/arch/tile/lib/atomic_asm_32.S index 6bda3132cd61..f611265633d6 100644 --- a/arch/tile/lib/atomic_asm_32.S +++ b/arch/tile/lib/atomic_asm_32.S @@ -178,6 +178,7 @@ atomic_op _xchg_add, 32, "add r24, r22, r2" atomic_op _xchg_add_unless, 32, \ "sne r26, r22, r2; { bbns r26, 3f; add r24, r22, r3 }" atomic_op _or, 32, "or r24, r22, r2" +atomic_op _and, 32, "and r24, r22, r2" atomic_op _andn, 32, "nor r2, r2, zero; and r24, r22, r2" atomic_op _xor, 32, "xor r24, r22, r2" @@ -191,6 +192,9 @@ atomic_op 64_xchg_add_unless, 64, \ { bbns r26, 3f; add r24, r22, r4 }; \ { bbns r27, 3f; add r25, r23, r5 }; \ slt_u r26, r24, r22; add r25, r25, r26" +atomic_op 64_or, 64, "{ or r24, r22, r2; or r25, r23, r3 }" +atomic_op 64_and, 64, "{ and r24, r22, r2; and r25, r23, r3 }" +atomic_op 64_xor, 64, "{ xor r24, r22, r2; xor r25, r23, r3 }" jrp lr /* happy backtracer */ -- cgit v1.2.3 From e6942b7de2dfe44ebde9bae57dadece5abca9de8 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 23 Apr 2014 19:32:50 +0200 Subject: atomic: Provide atomic_{or,xor,and} Implement atomic logic ops -- atomic_{or,xor,and}. These will replace the atomic_{set,clear}_mask functions that are available on some archs. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/alpha/include/asm/atomic.h | 1 - arch/arc/include/asm/atomic.h | 1 - arch/arm/include/asm/atomic.h | 1 - arch/arm64/include/asm/atomic.h | 1 - arch/avr32/include/asm/atomic.h | 2 -- arch/blackfin/include/asm/atomic.h | 2 -- arch/frv/include/asm/atomic.h | 2 -- arch/h8300/include/asm/atomic.h | 2 -- arch/hexagon/include/asm/atomic.h | 2 -- arch/ia64/include/asm/atomic.h | 2 -- arch/m32r/include/asm/atomic.h | 2 -- arch/m68k/include/asm/atomic.h | 2 -- arch/metag/include/asm/atomic_lnkget.h | 2 -- arch/mips/include/asm/atomic.h | 2 -- arch/mn10300/include/asm/atomic.h | 2 -- arch/parisc/include/asm/atomic.h | 2 -- arch/powerpc/include/asm/atomic.h | 2 -- arch/s390/include/asm/atomic.h | 2 -- arch/sh/include/asm/atomic-grb.h | 2 -- arch/sparc/include/asm/atomic_32.h | 2 -- arch/sparc/include/asm/atomic_64.h | 2 -- arch/tile/include/asm/atomic_32.h | 2 -- arch/tile/include/asm/atomic_64.h | 2 -- arch/x86/include/asm/atomic.h | 2 -- arch/xtensa/include/asm/atomic.h | 2 -- include/asm-generic/atomic.h | 21 ++++++++++++--------- include/asm-generic/atomic64.h | 4 ++++ include/linux/atomic.h | 13 ------------- lib/atomic64.c | 3 +++ 29 files changed, 19 insertions(+), 68 deletions(-) (limited to 'arch') diff --git a/arch/alpha/include/asm/atomic.h b/arch/alpha/include/asm/atomic.h index 0eff853398d2..e8c956098424 100644 --- a/arch/alpha/include/asm/atomic.h +++ b/arch/alpha/include/asm/atomic.h @@ -110,7 +110,6 @@ static __inline__ long atomic64_##op##_return(long i, atomic64_t * v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) -#define CONFIG_ARCH_HAS_ATOMIC_OR #define atomic_andnot atomic_andnot #define atomic64_andnot atomic64_andnot diff --git a/arch/arc/include/asm/atomic.h b/arch/arc/include/asm/atomic.h index e90b701fc6a8..2a847821dee1 100644 --- a/arch/arc/include/asm/atomic.h +++ b/arch/arc/include/asm/atomic.h @@ -144,7 +144,6 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add, +=, add) ATOMIC_OPS(sub, -=, sub) -#define CONFIG_ARCH_HAS_ATOMIC_OR #define atomic_andnot atomic_andnot ATOMIC_OP(and, &=, and) diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h index ff214bac9cb4..82b75a7cb762 100644 --- a/arch/arm/include/asm/atomic.h +++ b/arch/arm/include/asm/atomic.h @@ -194,7 +194,6 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) ATOMIC_OPS(add, +=, add) ATOMIC_OPS(sub, -=, sub) -#define CONFIG_ARCH_HAS_ATOMIC_OR #define atomic_andnot atomic_andnot ATOMIC_OP(and, &=, and) diff --git a/arch/arm64/include/asm/atomic.h b/arch/arm64/include/asm/atomic.h index 2876173397b2..866a71fca9a3 100644 --- a/arch/arm64/include/asm/atomic.h +++ b/arch/arm64/include/asm/atomic.h @@ -85,7 +85,6 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add, add) ATOMIC_OPS(sub, sub) -#define CONFIG_ARCH_HAS_ATOMIC_OR #define atomic_andnot atomic_andnot ATOMIC_OP(and, and) diff --git a/arch/avr32/include/asm/atomic.h b/arch/avr32/include/asm/atomic.h index 115d3005e4bc..97c9bdf83409 100644 --- a/arch/avr32/include/asm/atomic.h +++ b/arch/avr32/include/asm/atomic.h @@ -51,8 +51,6 @@ static inline void atomic_##op(int i, atomic_t *v) \ (void)__atomic_##op##_return(i, v); \ } -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(and, and) ATOMIC_OP(or, or) ATOMIC_OP(xor, eor) diff --git a/arch/blackfin/include/asm/atomic.h b/arch/blackfin/include/asm/atomic.h index eafa55b81a7b..2d6a7a3823c3 100644 --- a/arch/blackfin/include/asm/atomic.h +++ b/arch/blackfin/include/asm/atomic.h @@ -28,8 +28,6 @@ asmlinkage int __raw_atomic_test_asm(const volatile int *ptr, int value); #define atomic_add_return(i, v) __raw_atomic_add_asm(&(v)->counter, i) #define atomic_sub_return(i, v) __raw_atomic_add_asm(&(v)->counter, -(i)) -#define CONFIG_ARCH_HAS_ATOMIC_OR - #define atomic_or(i, v) (void)__raw_atomic_or_asm(&(v)->counter, i) #define atomic_and(i, v) (void)__raw_atomic_and_asm(&(v)->counter, i) #define atomic_xor(i, v) (void)__raw_atomic_xor_asm(&(v)->counter, i) diff --git a/arch/frv/include/asm/atomic.h b/arch/frv/include/asm/atomic.h index 74d22454d7c6..fc48bea26b40 100644 --- a/arch/frv/include/asm/atomic.h +++ b/arch/frv/include/asm/atomic.h @@ -192,8 +192,6 @@ static inline void atomic64_##op(long long i, atomic64_t *v) \ (void)__atomic64_fetch_##op(i, &v->counter); \ } -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(or) ATOMIC_OP(and) ATOMIC_OP(xor) diff --git a/arch/h8300/include/asm/atomic.h b/arch/h8300/include/asm/atomic.h index f181f820be33..c4d061f09c44 100644 --- a/arch/h8300/include/asm/atomic.h +++ b/arch/h8300/include/asm/atomic.h @@ -41,8 +41,6 @@ static inline void atomic_##op(int i, atomic_t *v) \ ATOMIC_OP_RETURN(add, +=) ATOMIC_OP_RETURN(sub, -=) -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(and, &=) ATOMIC_OP(or, |=) ATOMIC_OP(xor, ^=) diff --git a/arch/hexagon/include/asm/atomic.h b/arch/hexagon/include/asm/atomic.h index 4efe2c7c0dd8..811d61f6422d 100644 --- a/arch/hexagon/include/asm/atomic.h +++ b/arch/hexagon/include/asm/atomic.h @@ -132,8 +132,6 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(and) ATOMIC_OP(or) ATOMIC_OP(xor) diff --git a/arch/ia64/include/asm/atomic.h b/arch/ia64/include/asm/atomic.h index 0809ef5d6b9a..be4beeb77d57 100644 --- a/arch/ia64/include/asm/atomic.h +++ b/arch/ia64/include/asm/atomic.h @@ -69,8 +69,6 @@ ATOMIC_OP(sub, -) : ia64_atomic_sub(__ia64_asr_i, v); \ }) -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(and, &) ATOMIC_OP(or, |) ATOMIC_OP(xor, ^) diff --git a/arch/m32r/include/asm/atomic.h b/arch/m32r/include/asm/atomic.h index 7245463c1e98..b2a13fbd5be0 100644 --- a/arch/m32r/include/asm/atomic.h +++ b/arch/m32r/include/asm/atomic.h @@ -94,8 +94,6 @@ static __inline__ int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(and) ATOMIC_OP(or) ATOMIC_OP(xor) diff --git a/arch/m68k/include/asm/atomic.h b/arch/m68k/include/asm/atomic.h index c30e43ea49a3..93ebd96aa494 100644 --- a/arch/m68k/include/asm/atomic.h +++ b/arch/m68k/include/asm/atomic.h @@ -77,8 +77,6 @@ static inline int atomic_##op##_return(int i, atomic_t * v) \ ATOMIC_OPS(add, +=, add) ATOMIC_OPS(sub, -=, sub) -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(and, &=, and) ATOMIC_OP(or, |=, or) ATOMIC_OP(xor, ^=, eor) diff --git a/arch/metag/include/asm/atomic_lnkget.h b/arch/metag/include/asm/atomic_lnkget.h index 930c12cb8d37..0642606de901 100644 --- a/arch/metag/include/asm/atomic_lnkget.h +++ b/arch/metag/include/asm/atomic_lnkget.h @@ -74,8 +74,6 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(and) ATOMIC_OP(or) ATOMIC_OP(xor) diff --git a/arch/mips/include/asm/atomic.h b/arch/mips/include/asm/atomic.h index 0430ba6ab762..4c42fd9af777 100644 --- a/arch/mips/include/asm/atomic.h +++ b/arch/mips/include/asm/atomic.h @@ -137,8 +137,6 @@ static __inline__ int atomic_##op##_return(int i, atomic_t * v) \ ATOMIC_OPS(add, +=, addu) ATOMIC_OPS(sub, -=, subu) -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(and, &=, and) ATOMIC_OP(or, |=, or) ATOMIC_OP(xor, ^=, xor) diff --git a/arch/mn10300/include/asm/atomic.h b/arch/mn10300/include/asm/atomic.h index 03eea8158cf9..f5a63f0bda46 100644 --- a/arch/mn10300/include/asm/atomic.h +++ b/arch/mn10300/include/asm/atomic.h @@ -89,8 +89,6 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(and) ATOMIC_OP(or) ATOMIC_OP(xor) diff --git a/arch/parisc/include/asm/atomic.h b/arch/parisc/include/asm/atomic.h index be2c50ddebd6..2536965d00ea 100644 --- a/arch/parisc/include/asm/atomic.h +++ b/arch/parisc/include/asm/atomic.h @@ -126,8 +126,6 @@ static __inline__ int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add, +=) ATOMIC_OPS(sub, -=) -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(and, &=) ATOMIC_OP(or, |=) ATOMIC_OP(xor, ^=) diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h index 6ca89e2aca15..55f106ed12bf 100644 --- a/arch/powerpc/include/asm/atomic.h +++ b/arch/powerpc/include/asm/atomic.h @@ -67,8 +67,6 @@ static __inline__ int atomic_##op##_return(int a, atomic_t *v) \ ATOMIC_OPS(add, add) ATOMIC_OPS(sub, subf) -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(and, and) ATOMIC_OP(or, or) ATOMIC_OP(xor, xor) diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h index b3859d8e001f..d761aeff72da 100644 --- a/arch/s390/include/asm/atomic.h +++ b/arch/s390/include/asm/atomic.h @@ -282,8 +282,6 @@ static inline void atomic64_##op(long i, atomic64_t *v) \ __ATOMIC64_LOOP(v, i, __ATOMIC64_##OP, __ATOMIC64_NO_BARRIER); \ } -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC64_OP(and, AND) ATOMIC64_OP(or, OR) ATOMIC64_OP(xor, XOR) diff --git a/arch/sh/include/asm/atomic-grb.h b/arch/sh/include/asm/atomic-grb.h index 4b03830d48c7..b94df40e5f2d 100644 --- a/arch/sh/include/asm/atomic-grb.h +++ b/arch/sh/include/asm/atomic-grb.h @@ -48,8 +48,6 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(and) ATOMIC_OP(or) ATOMIC_OP(xor) diff --git a/arch/sparc/include/asm/atomic_32.h b/arch/sparc/include/asm/atomic_32.h index e19d8880b146..7dcbebbcaec6 100644 --- a/arch/sparc/include/asm/atomic_32.h +++ b/arch/sparc/include/asm/atomic_32.h @@ -17,8 +17,6 @@ #include #include -#define CONFIG_ARCH_HAS_ATOMIC_OR - #define ATOMIC_INIT(i) { (i) } int atomic_add_return(int, atomic_t *); diff --git a/arch/sparc/include/asm/atomic_64.h b/arch/sparc/include/asm/atomic_64.h index d6af27c93450..917084ace49d 100644 --- a/arch/sparc/include/asm/atomic_64.h +++ b/arch/sparc/include/asm/atomic_64.h @@ -33,8 +33,6 @@ long atomic64_##op##_return(long, atomic64_t *); ATOMIC_OPS(add) ATOMIC_OPS(sub) -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(and) ATOMIC_OP(or) ATOMIC_OP(xor) diff --git a/arch/tile/include/asm/atomic_32.h b/arch/tile/include/asm/atomic_32.h index 94237922f0dd..d320ce253d86 100644 --- a/arch/tile/include/asm/atomic_32.h +++ b/arch/tile/include/asm/atomic_32.h @@ -41,8 +41,6 @@ static inline void atomic_##op(int i, atomic_t *v) \ _atomic_##op((unsigned long *)&v->counter, i); \ } -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(and) ATOMIC_OP(or) ATOMIC_OP(xor) diff --git a/arch/tile/include/asm/atomic_64.h b/arch/tile/include/asm/atomic_64.h index d07d9fc6e2a1..096a56d6ead4 100644 --- a/arch/tile/include/asm/atomic_64.h +++ b/arch/tile/include/asm/atomic_64.h @@ -58,8 +58,6 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) return oldval; } -#define CONFIG_ARCH_HAS_ATOMIC_OR - static inline void atomic_and(int i, atomic_t *v) { __insn_fetchand4((void *)&v->counter, i); diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h index f3a3ec040694..b3493023efda 100644 --- a/arch/x86/include/asm/atomic.h +++ b/arch/x86/include/asm/atomic.h @@ -191,8 +191,6 @@ static inline void atomic_##op(int i, atomic_t *v) \ : "memory"); \ } -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(and) ATOMIC_OP(or) ATOMIC_OP(xor) diff --git a/arch/xtensa/include/asm/atomic.h b/arch/xtensa/include/asm/atomic.h index 4dd2450300a6..31371f43c23b 100644 --- a/arch/xtensa/include/asm/atomic.h +++ b/arch/xtensa/include/asm/atomic.h @@ -145,8 +145,6 @@ static inline int atomic_##op##_return(int i, atomic_t * v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) -#define CONFIG_ARCH_HAS_ATOMIC_OR - ATOMIC_OP(and) ATOMIC_OP(or) ATOMIC_OP(xor) diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h index 92947e0a532a..a41b0b8f7404 100644 --- a/include/asm-generic/atomic.h +++ b/include/asm-generic/atomic.h @@ -102,24 +102,27 @@ ATOMIC_OP_RETURN(sub, -) ATOMIC_OP(and, &) #endif -#ifndef atomic_clear_mask -#define atomic_clear_mask(i, v) atomic_and(~(i), (v)) -#endif - #ifndef atomic_or -#ifndef CONFIG_ARCH_HAS_ATOMIC_OR -#define CONFIG_ARCH_HAS_ATOMIC_OR -#endif ATOMIC_OP(or, |) #endif -#ifndef atomic_set_mask -#define atomic_set_mask(i, v) atomic_or((i), (v)) +#ifndef atomic_xor +ATOMIC_OP(xor, ^) #endif #undef ATOMIC_OP_RETURN #undef ATOMIC_OP +static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) +{ + atomic_and(~mask, v); +} + +static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) +{ + atomic_or(mask, v); +} + /* * Atomic operations that C can't guarantee us. Useful for * resource counting etc.. diff --git a/include/asm-generic/atomic64.h b/include/asm-generic/atomic64.h index 30ad9c86cebb..d48e78ccad3d 100644 --- a/include/asm-generic/atomic64.h +++ b/include/asm-generic/atomic64.h @@ -32,6 +32,10 @@ extern long long atomic64_##op##_return(long long a, atomic64_t *v); ATOMIC64_OPS(add) ATOMIC64_OPS(sub) +ATOMIC64_OP(and) +ATOMIC64_OP(or) +ATOMIC64_OP(xor) + #undef ATOMIC64_OPS #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP diff --git a/include/linux/atomic.h b/include/linux/atomic.h index 5b08a8540ecf..7d6279012a1f 100644 --- a/include/linux/atomic.h +++ b/include/linux/atomic.h @@ -111,19 +111,6 @@ static inline int atomic_dec_if_positive(atomic_t *v) } #endif -#ifndef CONFIG_ARCH_HAS_ATOMIC_OR -static inline void atomic_or(int i, atomic_t *v) -{ - int old; - int new; - - do { - old = atomic_read(v); - new = old | i; - } while (atomic_cmpxchg(v, old, new) != old); -} -#endif /* #ifndef CONFIG_ARCH_HAS_ATOMIC_OR */ - #include #ifdef CONFIG_GENERIC_ATOMIC64 #include diff --git a/lib/atomic64.c b/lib/atomic64.c index 1298c05ef528..2886ebac6567 100644 --- a/lib/atomic64.c +++ b/lib/atomic64.c @@ -102,6 +102,9 @@ EXPORT_SYMBOL(atomic64_##op##_return); ATOMIC64_OPS(add, +=) ATOMIC64_OPS(sub, -=) +ATOMIC64_OP(and, &=) +ATOMIC64_OP(or, |=) +ATOMIC64_OP(xor, ^=) #undef ATOMIC64_OPS #undef ATOMIC64_OP_RETURN -- cgit v1.2.3 From de9e432cb5de1bf2952919dc0b22e4bec0ed8d53 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 24 Apr 2015 01:12:32 +0200 Subject: atomic: Collapse all atomic_{set,clear}_mask definitions Move the now generic definitions of atomic_{set,clear}_mask() into linux/atomic.h to avoid endless and pointless repetition. Also, provide an atomic_andnot() wrapper for those few archs that can implement that. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/arc/include/asm/atomic.h | 10 ---------- arch/blackfin/include/asm/atomic.h | 10 ---------- arch/frv/include/asm/atomic.h | 10 ---------- arch/h8300/include/asm/atomic.h | 10 ---------- arch/m32r/include/asm/atomic.h | 11 ----------- arch/m68k/include/asm/atomic.h | 10 ---------- arch/metag/include/asm/atomic_lnkget.h | 10 ---------- arch/metag/include/asm/atomic_lock1.h | 10 ---------- arch/mn10300/include/asm/atomic.h | 24 ------------------------ arch/powerpc/kernel/misc_32.S | 19 ------------------- arch/s390/include/asm/atomic.h | 10 ---------- arch/sh/include/asm/atomic.h | 10 ---------- arch/x86/include/asm/atomic.h | 10 ---------- arch/xtensa/include/asm/atomic.h | 10 ---------- include/asm-generic/atomic.h | 10 ---------- include/linux/atomic.h | 25 +++++++++++++++++++++++++ 16 files changed, 25 insertions(+), 174 deletions(-) (limited to 'arch') diff --git a/arch/arc/include/asm/atomic.h b/arch/arc/include/asm/atomic.h index 2a847821dee1..d8a85e706fba 100644 --- a/arch/arc/include/asm/atomic.h +++ b/arch/arc/include/asm/atomic.h @@ -155,16 +155,6 @@ ATOMIC_OP(xor, ^=, xor) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - atomic_and(~mask, v); -} - -static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - atomic_or(mask, v); -} - /** * __atomic_add_unless - add unless the number is a given value * @v: pointer of type atomic_t diff --git a/arch/blackfin/include/asm/atomic.h b/arch/blackfin/include/asm/atomic.h index 2d6a7a3823c3..1c1c42330c99 100644 --- a/arch/blackfin/include/asm/atomic.h +++ b/arch/blackfin/include/asm/atomic.h @@ -32,16 +32,6 @@ asmlinkage int __raw_atomic_test_asm(const volatile int *ptr, int value); #define atomic_and(i, v) (void)__raw_atomic_and_asm(&(v)->counter, i) #define atomic_xor(i, v) (void)__raw_atomic_xor_asm(&(v)->counter, i) -static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - atomic_and(~mask, v); -} - -static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - atomic_or(mask, v); -} - #endif #include diff --git a/arch/frv/include/asm/atomic.h b/arch/frv/include/asm/atomic.h index fc48bea26b40..0da689def4cc 100644 --- a/arch/frv/include/asm/atomic.h +++ b/arch/frv/include/asm/atomic.h @@ -198,14 +198,4 @@ ATOMIC_OP(xor) #undef ATOMIC_OP -static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - atomic_and(~mask, v); -} - -static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - atomic_or(mask, v); -} - #endif /* _ASM_ATOMIC_H */ diff --git a/arch/h8300/include/asm/atomic.h b/arch/h8300/include/asm/atomic.h index c4d061f09c44..702ee539f87d 100644 --- a/arch/h8300/include/asm/atomic.h +++ b/arch/h8300/include/asm/atomic.h @@ -89,14 +89,4 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) return ret; } -static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - atomic_and(~mask, v); -} - -static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - atomic_or(mask, v); -} - #endif /* __ARCH_H8300_ATOMIC __ */ diff --git a/arch/m32r/include/asm/atomic.h b/arch/m32r/include/asm/atomic.h index b2a13fbd5be0..025e2a170493 100644 --- a/arch/m32r/include/asm/atomic.h +++ b/arch/m32r/include/asm/atomic.h @@ -243,15 +243,4 @@ static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) return c; } - -static __inline__ __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - atomic_and(~mask, v); -} - -static __inline__ __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - atomic_or(mask, v); -} - #endif /* _ASM_M32R_ATOMIC_H */ diff --git a/arch/m68k/include/asm/atomic.h b/arch/m68k/include/asm/atomic.h index 93ebd96aa494..039fac120cc0 100644 --- a/arch/m68k/include/asm/atomic.h +++ b/arch/m68k/include/asm/atomic.h @@ -174,16 +174,6 @@ static inline int atomic_add_negative(int i, atomic_t *v) return c != 0; } -static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - atomic_and(~mask, v); -} - -static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - atomic_or(mask, v); -} - static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) { int c, old; diff --git a/arch/metag/include/asm/atomic_lnkget.h b/arch/metag/include/asm/atomic_lnkget.h index 0642606de901..21c4c268b86c 100644 --- a/arch/metag/include/asm/atomic_lnkget.h +++ b/arch/metag/include/asm/atomic_lnkget.h @@ -82,16 +82,6 @@ ATOMIC_OP(xor) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - atomic_and(~mask, v); -} - -static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - atomic_or(mask, v); -} - static inline int atomic_cmpxchg(atomic_t *v, int old, int new) { int result, temp; diff --git a/arch/metag/include/asm/atomic_lock1.h b/arch/metag/include/asm/atomic_lock1.h index 7d88725a85da..f8efe380fe8b 100644 --- a/arch/metag/include/asm/atomic_lock1.h +++ b/arch/metag/include/asm/atomic_lock1.h @@ -76,16 +76,6 @@ ATOMIC_OP(xor, ^=) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - atomic_and(~mask, v); -} - -static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - atomic_or(mask, v); -} - static inline int atomic_cmpxchg(atomic_t *v, int old, int new) { int ret; diff --git a/arch/mn10300/include/asm/atomic.h b/arch/mn10300/include/asm/atomic.h index f5a63f0bda46..375e59140c9c 100644 --- a/arch/mn10300/include/asm/atomic.h +++ b/arch/mn10300/include/asm/atomic.h @@ -131,30 +131,6 @@ static inline void atomic_dec(atomic_t *v) #define atomic_xchg(ptr, v) (xchg(&(ptr)->counter, (v))) #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), (old), (new))) -/** - * atomic_clear_mask - Atomically clear bits in memory - * @mask: Mask of the bits to be cleared - * @v: pointer to word in memory - * - * Atomically clears the bits set in mask from the memory word specified. - */ -static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - atomic_and(~mask, v); -} - -/** - * atomic_set_mask - Atomically set bits in memory - * @mask: Mask of the bits to be set - * @v: pointer to word in memory - * - * Atomically sets the bits set in mask from the memory word specified. - */ -static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - atomic_or(mask, v); -} - #endif /* __KERNEL__ */ #endif /* CONFIG_SMP */ #endif /* _ASM_ATOMIC_H */ diff --git a/arch/powerpc/kernel/misc_32.S b/arch/powerpc/kernel/misc_32.S index 7c6bb4b17b49..ed3ab509faca 100644 --- a/arch/powerpc/kernel/misc_32.S +++ b/arch/powerpc/kernel/misc_32.S @@ -595,25 +595,6 @@ _GLOBAL(copy_page) li r11,4 b 2b -/* - * void atomic_clear_mask(atomic_t mask, atomic_t *addr) - * void atomic_set_mask(atomic_t mask, atomic_t *addr); - */ -_GLOBAL(atomic_clear_mask) -10: lwarx r5,0,r4 - andc r5,r5,r3 - PPC405_ERR77(0,r4) - stwcx. r5,0,r4 - bne- 10b - blr -_GLOBAL(atomic_set_mask) -10: lwarx r5,0,r4 - or r5,r5,r3 - PPC405_ERR77(0,r4) - stwcx. r5,0,r4 - bne- 10b - blr - /* * Extended precision shifts. * diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h index d761aeff72da..117fa5c921c1 100644 --- a/arch/s390/include/asm/atomic.h +++ b/arch/s390/include/asm/atomic.h @@ -132,16 +132,6 @@ ATOMIC_OP(xor, XOR) #undef ATOMIC_OP -static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - atomic_and(~mask, v); -} - -static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - atomic_or(mask, v); -} - #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) static inline int atomic_cmpxchg(atomic_t *v, int old, int new) diff --git a/arch/sh/include/asm/atomic.h b/arch/sh/include/asm/atomic.h index cee0245257e1..05b9f74ce2d5 100644 --- a/arch/sh/include/asm/atomic.h +++ b/arch/sh/include/asm/atomic.h @@ -25,16 +25,6 @@ #include #endif -static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - atomic_and(~mask, v); -} - -static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - atomic_or(mask, v); -} - #define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) #define atomic_dec_return(v) atomic_sub_return(1, (v)) #define atomic_inc_return(v) atomic_add_return(1, (v)) diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h index b3493023efda..fb52aa644aab 100644 --- a/arch/x86/include/asm/atomic.h +++ b/arch/x86/include/asm/atomic.h @@ -234,16 +234,6 @@ static __always_inline short int atomic_inc_short(short int *v) return *v; } -static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - atomic_and(~mask, v); -} - -static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - atomic_or(mask, v); -} - #ifdef CONFIG_X86_32 # include #else diff --git a/arch/xtensa/include/asm/atomic.h b/arch/xtensa/include/asm/atomic.h index 31371f43c23b..e0be67936990 100644 --- a/arch/xtensa/include/asm/atomic.h +++ b/arch/xtensa/include/asm/atomic.h @@ -153,16 +153,6 @@ ATOMIC_OP(xor) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - atomic_or(mask, v); -} - -static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - atomic_and(~mask, v); -} - /** * atomic_sub_and_test - subtract value from variable and test result * @i: integer value to subtract diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h index a41b0b8f7404..d4d7e337fdcb 100644 --- a/include/asm-generic/atomic.h +++ b/include/asm-generic/atomic.h @@ -113,16 +113,6 @@ ATOMIC_OP(xor, ^) #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - atomic_and(~mask, v); -} - -static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - atomic_or(mask, v); -} - /* * Atomic operations that C can't guarantee us. Useful for * resource counting etc.. diff --git a/include/linux/atomic.h b/include/linux/atomic.h index 7d6279012a1f..8b98b423388f 100644 --- a/include/linux/atomic.h +++ b/include/linux/atomic.h @@ -28,6 +28,23 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #endif +#ifndef atomic_andnot +static inline void atomic_andnot(int i, atomic_t *v) +{ + atomic_and(~i, v); +} +#endif + +static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) +{ + atomic_andnot(mask, v); +} + +static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) +{ + atomic_or(mask, v); +} + /** * atomic_inc_not_zero_hint - increment if not null * @v: pointer of type atomic_t @@ -115,4 +132,12 @@ static inline int atomic_dec_if_positive(atomic_t *v) #ifdef CONFIG_GENERIC_ATOMIC64 #include #endif + +#ifndef atomic64_andnot +static inline void atomic64_andnot(long long i, atomic64_t *v) +{ + atomic64_and(~i, v); +} +#endif + #endif /* _LINUX_ATOMIC_H */ -- cgit v1.2.3 From 805de8f43c20ba8b479bb598b543fa86b20067f6 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 24 Apr 2015 01:12:32 +0200 Subject: atomic: Replace atomic_{set,clear}_mask() usage Replace the deprecated atomic_{set,clear}_mask() usage with the now ubiquous atomic_{or,andnot}() functions. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner --- arch/blackfin/mach-common/smp.c | 2 +- arch/m32r/kernel/smp.c | 4 +-- arch/mn10300/mm/tlb-smp.c | 2 +- arch/s390/kernel/time.c | 4 +-- arch/s390/kvm/interrupt.c | 30 ++++++++++---------- arch/s390/kvm/kvm-s390.c | 32 ++++++++++----------- drivers/gpu/drm/i915/i915_drv.c | 2 +- drivers/gpu/drm/i915/i915_gem.c | 2 +- drivers/gpu/drm/i915/i915_irq.c | 4 +-- drivers/s390/scsi/zfcp_aux.c | 2 +- drivers/s390/scsi/zfcp_erp.c | 62 ++++++++++++++++++++--------------------- drivers/s390/scsi/zfcp_fc.c | 8 +++--- drivers/s390/scsi/zfcp_fsf.c | 26 ++++++++--------- drivers/s390/scsi/zfcp_qdio.c | 14 +++++----- 14 files changed, 97 insertions(+), 97 deletions(-) (limited to 'arch') diff --git a/arch/blackfin/mach-common/smp.c b/arch/blackfin/mach-common/smp.c index 1c7259597395..0030e21cfceb 100644 --- a/arch/blackfin/mach-common/smp.c +++ b/arch/blackfin/mach-common/smp.c @@ -195,7 +195,7 @@ void send_ipi(const struct cpumask *cpumask, enum ipi_message_type msg) local_irq_save(flags); for_each_cpu(cpu, cpumask) { bfin_ipi_data = &per_cpu(bfin_ipi, cpu); - atomic_set_mask((1 << msg), &bfin_ipi_data->bits); + atomic_or((1 << msg), &bfin_ipi_data->bits); atomic_inc(&bfin_ipi_data->count); } local_irq_restore(flags); diff --git a/arch/m32r/kernel/smp.c b/arch/m32r/kernel/smp.c index c18ddc74ef9a..62d6961e7f2b 100644 --- a/arch/m32r/kernel/smp.c +++ b/arch/m32r/kernel/smp.c @@ -156,7 +156,7 @@ void smp_flush_cache_all(void) cpumask_clear_cpu(smp_processor_id(), &cpumask); spin_lock(&flushcache_lock); mask=cpumask_bits(&cpumask); - atomic_set_mask(*mask, (atomic_t *)&flushcache_cpumask); + atomic_or(*mask, (atomic_t *)&flushcache_cpumask); send_IPI_mask(&cpumask, INVALIDATE_CACHE_IPI, 0); _flush_cache_copyback_all(); while (flushcache_cpumask) @@ -407,7 +407,7 @@ static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, flush_vma = vma; flush_va = va; mask=cpumask_bits(&cpumask); - atomic_set_mask(*mask, (atomic_t *)&flush_cpumask); + atomic_or(*mask, (atomic_t *)&flush_cpumask); /* * We have to send the IPI only to diff --git a/arch/mn10300/mm/tlb-smp.c b/arch/mn10300/mm/tlb-smp.c index e5d0ef722bfa..9a39ea9031d4 100644 --- a/arch/mn10300/mm/tlb-smp.c +++ b/arch/mn10300/mm/tlb-smp.c @@ -119,7 +119,7 @@ static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, flush_mm = mm; flush_va = va; #if NR_CPUS <= BITS_PER_LONG - atomic_set_mask(cpumask.bits[0], &flush_cpumask.bits[0]); + atomic_or(cpumask.bits[0], (atomic_t *)&flush_cpumask.bits[0]); #else #error Not supported. #endif diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 9e733d965e08..f5a0bd778ace 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -381,7 +381,7 @@ static void disable_sync_clock(void *dummy) * increase the "sequence" counter to avoid the race of an * etr event and the complete recovery against get_sync_clock. */ - atomic_clear_mask(0x80000000, sw_ptr); + atomic_andnot(0x80000000, sw_ptr); atomic_inc(sw_ptr); } @@ -392,7 +392,7 @@ static void disable_sync_clock(void *dummy) static void enable_sync_clock(void) { atomic_t *sw_ptr = this_cpu_ptr(&clock_sync_word); - atomic_set_mask(0x80000000, sw_ptr); + atomic_or(0x80000000, sw_ptr); } /* diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index c98d89708e99..57309e9cdd80 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -170,20 +170,20 @@ static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu) static void __set_cpu_idle(struct kvm_vcpu *vcpu) { - atomic_set_mask(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); + atomic_or(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); set_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask); } static void __unset_cpu_idle(struct kvm_vcpu *vcpu) { - atomic_clear_mask(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); + atomic_andnot(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); clear_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask); } static void __reset_intercept_indicators(struct kvm_vcpu *vcpu) { - atomic_clear_mask(CPUSTAT_IO_INT | CPUSTAT_EXT_INT | CPUSTAT_STOP_INT, - &vcpu->arch.sie_block->cpuflags); + atomic_andnot(CPUSTAT_IO_INT | CPUSTAT_EXT_INT | CPUSTAT_STOP_INT, + &vcpu->arch.sie_block->cpuflags); vcpu->arch.sie_block->lctl = 0x0000; vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT); @@ -196,7 +196,7 @@ static void __reset_intercept_indicators(struct kvm_vcpu *vcpu) static void __set_cpuflag(struct kvm_vcpu *vcpu, u32 flag) { - atomic_set_mask(flag, &vcpu->arch.sie_block->cpuflags); + atomic_or(flag, &vcpu->arch.sie_block->cpuflags); } static void set_intercept_indicators_io(struct kvm_vcpu *vcpu) @@ -919,7 +919,7 @@ void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu) spin_unlock(&li->lock); /* clear pending external calls set by sigp interpretation facility */ - atomic_clear_mask(CPUSTAT_ECALL_PEND, li->cpuflags); + atomic_andnot(CPUSTAT_ECALL_PEND, li->cpuflags); vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sigp_ctrl = 0; } @@ -1020,7 +1020,7 @@ static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) li->irq.ext = irq->u.ext; set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs); - atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags); + atomic_or(CPUSTAT_EXT_INT, li->cpuflags); return 0; } @@ -1035,7 +1035,7 @@ static int __inject_extcall_sigpif(struct kvm_vcpu *vcpu, uint16_t src_id) /* another external call is pending */ return -EBUSY; } - atomic_set_mask(CPUSTAT_ECALL_PEND, &vcpu->arch.sie_block->cpuflags); + atomic_or(CPUSTAT_ECALL_PEND, &vcpu->arch.sie_block->cpuflags); return 0; } @@ -1061,7 +1061,7 @@ static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs)) return -EBUSY; *extcall = irq->u.extcall; - atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags); + atomic_or(CPUSTAT_EXT_INT, li->cpuflags); return 0; } @@ -1133,7 +1133,7 @@ static int __inject_sigp_emergency(struct kvm_vcpu *vcpu, set_bit(irq->u.emerg.code, li->sigp_emerg_pending); set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs); - atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags); + atomic_or(CPUSTAT_EXT_INT, li->cpuflags); return 0; } @@ -1177,7 +1177,7 @@ static int __inject_ckc(struct kvm_vcpu *vcpu) 0, 0, 2); set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); - atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags); + atomic_or(CPUSTAT_EXT_INT, li->cpuflags); return 0; } @@ -1190,7 +1190,7 @@ static int __inject_cpu_timer(struct kvm_vcpu *vcpu) 0, 0, 2); set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); - atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags); + atomic_or(CPUSTAT_EXT_INT, li->cpuflags); return 0; } @@ -1369,13 +1369,13 @@ static void __floating_irq_kick(struct kvm *kvm, u64 type) spin_lock(&li->lock); switch (type) { case KVM_S390_MCHK: - atomic_set_mask(CPUSTAT_STOP_INT, li->cpuflags); + atomic_or(CPUSTAT_STOP_INT, li->cpuflags); break; case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: - atomic_set_mask(CPUSTAT_IO_INT, li->cpuflags); + atomic_or(CPUSTAT_IO_INT, li->cpuflags); break; default: - atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags); + atomic_or(CPUSTAT_EXT_INT, li->cpuflags); break; } spin_unlock(&li->lock); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 2078f92d15ac..b73302fb0507 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -1215,12 +1215,12 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) } restore_access_regs(vcpu->run->s.regs.acrs); gmap_enable(vcpu->arch.gmap); - atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); + atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); } void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) { - atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); + atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); gmap_disable(vcpu->arch.gmap); if (test_kvm_facility(vcpu->kvm, 129)) { save_fp_ctl(&vcpu->run->s.regs.fpc); @@ -1320,9 +1320,9 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) CPUSTAT_STOPPED); if (test_kvm_facility(vcpu->kvm, 78)) - atomic_set_mask(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags); + atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags); else if (test_kvm_facility(vcpu->kvm, 8)) - atomic_set_mask(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags); + atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags); kvm_s390_vcpu_setup_model(vcpu); @@ -1422,24 +1422,24 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu) { - atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); + atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); exit_sie(vcpu); } void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu) { - atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); + atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); } static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu) { - atomic_set_mask(PROG_REQUEST, &vcpu->arch.sie_block->prog20); + atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20); exit_sie(vcpu); } static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu) { - atomic_clear_mask(PROG_REQUEST, &vcpu->arch.sie_block->prog20); + atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20); } /* @@ -1448,7 +1448,7 @@ static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu) * return immediately. */ void exit_sie(struct kvm_vcpu *vcpu) { - atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags); + atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags); while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE) cpu_relax(); } @@ -1672,19 +1672,19 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, if (dbg->control & KVM_GUESTDBG_ENABLE) { vcpu->guest_debug = dbg->control; /* enforce guest PER */ - atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); + atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); if (dbg->control & KVM_GUESTDBG_USE_HW_BP) rc = kvm_s390_import_bp_data(vcpu, dbg); } else { - atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); + atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); vcpu->arch.guestdbg.last_bp = 0; } if (rc) { vcpu->guest_debug = 0; kvm_s390_clear_bp_data(vcpu); - atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); + atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); } return rc; @@ -1771,7 +1771,7 @@ retry: if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) { if (!ibs_enabled(vcpu)) { trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1); - atomic_set_mask(CPUSTAT_IBS, + atomic_or(CPUSTAT_IBS, &vcpu->arch.sie_block->cpuflags); } goto retry; @@ -1780,7 +1780,7 @@ retry: if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) { if (ibs_enabled(vcpu)) { trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0); - atomic_clear_mask(CPUSTAT_IBS, + atomic_andnot(CPUSTAT_IBS, &vcpu->arch.sie_block->cpuflags); } goto retry; @@ -2280,7 +2280,7 @@ void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu) __disable_ibs_on_all_vcpus(vcpu->kvm); } - atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); + atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); /* * Another VCPU might have used IBS while we were offline. * Let's play safe and flush the VCPU at startup. @@ -2306,7 +2306,7 @@ void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu) /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */ kvm_s390_clear_stop_irq(vcpu); - atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); + atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); __disable_ibs_on_vcpu(vcpu); for (i = 0; i < online_vcpus; i++) { diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 884b4f9b81c4..8917c98ff121 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -748,7 +748,7 @@ static int i915_drm_resume(struct drm_device *dev) mutex_lock(&dev->struct_mutex); if (i915_gem_init_hw(dev)) { DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n"); - atomic_set_mask(I915_WEDGED, &dev_priv->gpu_error.reset_counter); + atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter); } mutex_unlock(&dev->struct_mutex); diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 52b446b27b4d..7a918d1c12ba 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -5091,7 +5091,7 @@ int i915_gem_init(struct drm_device *dev) * for all other failure, such as an allocation failure, bail. */ DRM_ERROR("Failed to initialize GPU, declaring it wedged\n"); - atomic_set_mask(I915_WEDGED, &dev_priv->gpu_error.reset_counter); + atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter); ret = 0; } diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 984e2fe6688c..449a95c6c2a1 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -2446,7 +2446,7 @@ static void i915_reset_and_wakeup(struct drm_device *dev) kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, reset_done_event); } else { - atomic_set_mask(I915_WEDGED, &error->reset_counter); + atomic_or(I915_WEDGED, &error->reset_counter); } /* @@ -2574,7 +2574,7 @@ void i915_handle_error(struct drm_device *dev, bool wedged, i915_report_and_clear_eir(dev); if (wedged) { - atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG, + atomic_or(I915_RESET_IN_PROGRESS_FLAG, &dev_priv->gpu_error.reset_counter); /* diff --git a/drivers/s390/scsi/zfcp_aux.c b/drivers/s390/scsi/zfcp_aux.c index 01a73395a017..c00ac4650dce 100644 --- a/drivers/s390/scsi/zfcp_aux.c +++ b/drivers/s390/scsi/zfcp_aux.c @@ -529,7 +529,7 @@ struct zfcp_port *zfcp_port_enqueue(struct zfcp_adapter *adapter, u64 wwpn, list_add_tail(&port->list, &adapter->port_list); write_unlock_irq(&adapter->port_list_lock); - atomic_set_mask(status | ZFCP_STATUS_COMMON_RUNNING, &port->status); + atomic_or(status | ZFCP_STATUS_COMMON_RUNNING, &port->status); return port; diff --git a/drivers/s390/scsi/zfcp_erp.c b/drivers/s390/scsi/zfcp_erp.c index acde3f5d6e9e..3fb410977014 100644 --- a/drivers/s390/scsi/zfcp_erp.c +++ b/drivers/s390/scsi/zfcp_erp.c @@ -190,7 +190,7 @@ static struct zfcp_erp_action *zfcp_erp_setup_act(int need, u32 act_status, if (!(act_status & ZFCP_STATUS_ERP_NO_REF)) if (scsi_device_get(sdev)) return NULL; - atomic_set_mask(ZFCP_STATUS_COMMON_ERP_INUSE, + atomic_or(ZFCP_STATUS_COMMON_ERP_INUSE, &zfcp_sdev->status); erp_action = &zfcp_sdev->erp_action; memset(erp_action, 0, sizeof(struct zfcp_erp_action)); @@ -206,7 +206,7 @@ static struct zfcp_erp_action *zfcp_erp_setup_act(int need, u32 act_status, if (!get_device(&port->dev)) return NULL; zfcp_erp_action_dismiss_port(port); - atomic_set_mask(ZFCP_STATUS_COMMON_ERP_INUSE, &port->status); + atomic_or(ZFCP_STATUS_COMMON_ERP_INUSE, &port->status); erp_action = &port->erp_action; memset(erp_action, 0, sizeof(struct zfcp_erp_action)); erp_action->port = port; @@ -217,7 +217,7 @@ static struct zfcp_erp_action *zfcp_erp_setup_act(int need, u32 act_status, case ZFCP_ERP_ACTION_REOPEN_ADAPTER: kref_get(&adapter->ref); zfcp_erp_action_dismiss_adapter(adapter); - atomic_set_mask(ZFCP_STATUS_COMMON_ERP_INUSE, &adapter->status); + atomic_or(ZFCP_STATUS_COMMON_ERP_INUSE, &adapter->status); erp_action = &adapter->erp_action; memset(erp_action, 0, sizeof(struct zfcp_erp_action)); if (!(atomic_read(&adapter->status) & @@ -254,7 +254,7 @@ static int zfcp_erp_action_enqueue(int want, struct zfcp_adapter *adapter, act = zfcp_erp_setup_act(need, act_status, adapter, port, sdev); if (!act) goto out; - atomic_set_mask(ZFCP_STATUS_ADAPTER_ERP_PENDING, &adapter->status); + atomic_or(ZFCP_STATUS_ADAPTER_ERP_PENDING, &adapter->status); ++adapter->erp_total_count; list_add_tail(&act->list, &adapter->erp_ready_head); wake_up(&adapter->erp_ready_wq); @@ -486,14 +486,14 @@ static void zfcp_erp_adapter_unblock(struct zfcp_adapter *adapter) { if (status_change_set(ZFCP_STATUS_COMMON_UNBLOCKED, &adapter->status)) zfcp_dbf_rec_run("eraubl1", &adapter->erp_action); - atomic_set_mask(ZFCP_STATUS_COMMON_UNBLOCKED, &adapter->status); + atomic_or(ZFCP_STATUS_COMMON_UNBLOCKED, &adapter->status); } static void zfcp_erp_port_unblock(struct zfcp_port *port) { if (status_change_set(ZFCP_STATUS_COMMON_UNBLOCKED, &port->status)) zfcp_dbf_rec_run("erpubl1", &port->erp_action); - atomic_set_mask(ZFCP_STATUS_COMMON_UNBLOCKED, &port->status); + atomic_or(ZFCP_STATUS_COMMON_UNBLOCKED, &port->status); } static void zfcp_erp_lun_unblock(struct scsi_device *sdev) @@ -502,7 +502,7 @@ static void zfcp_erp_lun_unblock(struct scsi_device *sdev) if (status_change_set(ZFCP_STATUS_COMMON_UNBLOCKED, &zfcp_sdev->status)) zfcp_dbf_rec_run("erlubl1", &sdev_to_zfcp(sdev)->erp_action); - atomic_set_mask(ZFCP_STATUS_COMMON_UNBLOCKED, &zfcp_sdev->status); + atomic_or(ZFCP_STATUS_COMMON_UNBLOCKED, &zfcp_sdev->status); } static void zfcp_erp_action_to_running(struct zfcp_erp_action *erp_action) @@ -642,7 +642,7 @@ static void zfcp_erp_wakeup(struct zfcp_adapter *adapter) read_lock_irqsave(&adapter->erp_lock, flags); if (list_empty(&adapter->erp_ready_head) && list_empty(&adapter->erp_running_head)) { - atomic_clear_mask(ZFCP_STATUS_ADAPTER_ERP_PENDING, + atomic_andnot(ZFCP_STATUS_ADAPTER_ERP_PENDING, &adapter->status); wake_up(&adapter->erp_done_wqh); } @@ -665,16 +665,16 @@ static int zfcp_erp_adapter_strat_fsf_xconf(struct zfcp_erp_action *erp_action) int sleep = 1; struct zfcp_adapter *adapter = erp_action->adapter; - atomic_clear_mask(ZFCP_STATUS_ADAPTER_XCONFIG_OK, &adapter->status); + atomic_andnot(ZFCP_STATUS_ADAPTER_XCONFIG_OK, &adapter->status); for (retries = 7; retries; retries--) { - atomic_clear_mask(ZFCP_STATUS_ADAPTER_HOST_CON_INIT, + atomic_andnot(ZFCP_STATUS_ADAPTER_HOST_CON_INIT, &adapter->status); write_lock_irq(&adapter->erp_lock); zfcp_erp_action_to_running(erp_action); write_unlock_irq(&adapter->erp_lock); if (zfcp_fsf_exchange_config_data(erp_action)) { - atomic_clear_mask(ZFCP_STATUS_ADAPTER_HOST_CON_INIT, + atomic_andnot(ZFCP_STATUS_ADAPTER_HOST_CON_INIT, &adapter->status); return ZFCP_ERP_FAILED; } @@ -692,7 +692,7 @@ static int zfcp_erp_adapter_strat_fsf_xconf(struct zfcp_erp_action *erp_action) sleep *= 2; } - atomic_clear_mask(ZFCP_STATUS_ADAPTER_HOST_CON_INIT, + atomic_andnot(ZFCP_STATUS_ADAPTER_HOST_CON_INIT, &adapter->status); if (!(atomic_read(&adapter->status) & ZFCP_STATUS_ADAPTER_XCONFIG_OK)) @@ -764,7 +764,7 @@ static void zfcp_erp_adapter_strategy_close(struct zfcp_erp_action *act) /* all ports and LUNs are closed */ zfcp_erp_clear_adapter_status(adapter, ZFCP_STATUS_COMMON_OPEN); - atomic_clear_mask(ZFCP_STATUS_ADAPTER_XCONFIG_OK | + atomic_andnot(ZFCP_STATUS_ADAPTER_XCONFIG_OK | ZFCP_STATUS_ADAPTER_LINK_UNPLUGGED, &adapter->status); } @@ -773,7 +773,7 @@ static int zfcp_erp_adapter_strategy_open(struct zfcp_erp_action *act) struct zfcp_adapter *adapter = act->adapter; if (zfcp_qdio_open(adapter->qdio)) { - atomic_clear_mask(ZFCP_STATUS_ADAPTER_XCONFIG_OK | + atomic_andnot(ZFCP_STATUS_ADAPTER_XCONFIG_OK | ZFCP_STATUS_ADAPTER_LINK_UNPLUGGED, &adapter->status); return ZFCP_ERP_FAILED; @@ -784,7 +784,7 @@ static int zfcp_erp_adapter_strategy_open(struct zfcp_erp_action *act) return ZFCP_ERP_FAILED; } - atomic_set_mask(ZFCP_STATUS_COMMON_OPEN, &adapter->status); + atomic_or(ZFCP_STATUS_COMMON_OPEN, &adapter->status); return ZFCP_ERP_SUCCEEDED; } @@ -948,7 +948,7 @@ static void zfcp_erp_lun_strategy_clearstati(struct scsi_device *sdev) { struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(sdev); - atomic_clear_mask(ZFCP_STATUS_COMMON_ACCESS_DENIED, + atomic_andnot(ZFCP_STATUS_COMMON_ACCESS_DENIED, &zfcp_sdev->status); } @@ -1187,18 +1187,18 @@ static void zfcp_erp_action_dequeue(struct zfcp_erp_action *erp_action) switch (erp_action->action) { case ZFCP_ERP_ACTION_REOPEN_LUN: zfcp_sdev = sdev_to_zfcp(erp_action->sdev); - atomic_clear_mask(ZFCP_STATUS_COMMON_ERP_INUSE, + atomic_andnot(ZFCP_STATUS_COMMON_ERP_INUSE, &zfcp_sdev->status); break; case ZFCP_ERP_ACTION_REOPEN_PORT_FORCED: case ZFCP_ERP_ACTION_REOPEN_PORT: - atomic_clear_mask(ZFCP_STATUS_COMMON_ERP_INUSE, + atomic_andnot(ZFCP_STATUS_COMMON_ERP_INUSE, &erp_action->port->status); break; case ZFCP_ERP_ACTION_REOPEN_ADAPTER: - atomic_clear_mask(ZFCP_STATUS_COMMON_ERP_INUSE, + atomic_andnot(ZFCP_STATUS_COMMON_ERP_INUSE, &erp_action->adapter->status); break; } @@ -1422,19 +1422,19 @@ void zfcp_erp_set_adapter_status(struct zfcp_adapter *adapter, u32 mask) unsigned long flags; u32 common_mask = mask & ZFCP_COMMON_FLAGS; - atomic_set_mask(mask, &adapter->status); + atomic_or(mask, &adapter->status); if (!common_mask) return; read_lock_irqsave(&adapter->port_list_lock, flags); list_for_each_entry(port, &adapter->port_list, list) - atomic_set_mask(common_mask, &port->status); + atomic_or(common_mask, &port->status); read_unlock_irqrestore(&adapter->port_list_lock, flags); spin_lock_irqsave(adapter->scsi_host->host_lock, flags); __shost_for_each_device(sdev, adapter->scsi_host) - atomic_set_mask(common_mask, &sdev_to_zfcp(sdev)->status); + atomic_or(common_mask, &sdev_to_zfcp(sdev)->status); spin_unlock_irqrestore(adapter->scsi_host->host_lock, flags); } @@ -1453,7 +1453,7 @@ void zfcp_erp_clear_adapter_status(struct zfcp_adapter *adapter, u32 mask) u32 common_mask = mask & ZFCP_COMMON_FLAGS; u32 clear_counter = mask & ZFCP_STATUS_COMMON_ERP_FAILED; - atomic_clear_mask(mask, &adapter->status); + atomic_andnot(mask, &adapter->status); if (!common_mask) return; @@ -1463,7 +1463,7 @@ void zfcp_erp_clear_adapter_status(struct zfcp_adapter *adapter, u32 mask) read_lock_irqsave(&adapter->port_list_lock, flags); list_for_each_entry(port, &adapter->port_list, list) { - atomic_clear_mask(common_mask, &port->status); + atomic_andnot(common_mask, &port->status); if (clear_counter) atomic_set(&port->erp_counter, 0); } @@ -1471,7 +1471,7 @@ void zfcp_erp_clear_adapter_status(struct zfcp_adapter *adapter, u32 mask) spin_lock_irqsave(adapter->scsi_host->host_lock, flags); __shost_for_each_device(sdev, adapter->scsi_host) { - atomic_clear_mask(common_mask, &sdev_to_zfcp(sdev)->status); + atomic_andnot(common_mask, &sdev_to_zfcp(sdev)->status); if (clear_counter) atomic_set(&sdev_to_zfcp(sdev)->erp_counter, 0); } @@ -1491,7 +1491,7 @@ void zfcp_erp_set_port_status(struct zfcp_port *port, u32 mask) u32 common_mask = mask & ZFCP_COMMON_FLAGS; unsigned long flags; - atomic_set_mask(mask, &port->status); + atomic_or(mask, &port->status); if (!common_mask) return; @@ -1499,7 +1499,7 @@ void zfcp_erp_set_port_status(struct zfcp_port *port, u32 mask) spin_lock_irqsave(port->adapter->scsi_host->host_lock, flags); __shost_for_each_device(sdev, port->adapter->scsi_host) if (sdev_to_zfcp(sdev)->port == port) - atomic_set_mask(common_mask, + atomic_or(common_mask, &sdev_to_zfcp(sdev)->status); spin_unlock_irqrestore(port->adapter->scsi_host->host_lock, flags); } @@ -1518,7 +1518,7 @@ void zfcp_erp_clear_port_status(struct zfcp_port *port, u32 mask) u32 clear_counter = mask & ZFCP_STATUS_COMMON_ERP_FAILED; unsigned long flags; - atomic_clear_mask(mask, &port->status); + atomic_andnot(mask, &port->status); if (!common_mask) return; @@ -1529,7 +1529,7 @@ void zfcp_erp_clear_port_status(struct zfcp_port *port, u32 mask) spin_lock_irqsave(port->adapter->scsi_host->host_lock, flags); __shost_for_each_device(sdev, port->adapter->scsi_host) if (sdev_to_zfcp(sdev)->port == port) { - atomic_clear_mask(common_mask, + atomic_andnot(common_mask, &sdev_to_zfcp(sdev)->status); if (clear_counter) atomic_set(&sdev_to_zfcp(sdev)->erp_counter, 0); @@ -1546,7 +1546,7 @@ void zfcp_erp_set_lun_status(struct scsi_device *sdev, u32 mask) { struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(sdev); - atomic_set_mask(mask, &zfcp_sdev->status); + atomic_or(mask, &zfcp_sdev->status); } /** @@ -1558,7 +1558,7 @@ void zfcp_erp_clear_lun_status(struct scsi_device *sdev, u32 mask) { struct zfcp_scsi_dev *zfcp_sdev = sdev_to_zfcp(sdev); - atomic_clear_mask(mask, &zfcp_sdev->status); + atomic_andnot(mask, &zfcp_sdev->status); if (mask & ZFCP_STATUS_COMMON_ERP_FAILED) atomic_set(&zfcp_sdev->erp_counter, 0); diff --git a/drivers/s390/scsi/zfcp_fc.c b/drivers/s390/scsi/zfcp_fc.c index 25d49f32ca63..237688af179b 100644 --- a/drivers/s390/scsi/zfcp_fc.c +++ b/drivers/s390/scsi/zfcp_fc.c @@ -508,7 +508,7 @@ static void zfcp_fc_adisc_handler(void *data) /* port is good, unblock rport without going through erp */ zfcp_scsi_schedule_rport_register(port); out: - atomic_clear_mask(ZFCP_STATUS_PORT_LINK_TEST, &port->status); + atomic_andnot(ZFCP_STATUS_PORT_LINK_TEST, &port->status); put_device(&port->dev); kmem_cache_free(zfcp_fc_req_cache, fc_req); } @@ -564,14 +564,14 @@ void zfcp_fc_link_test_work(struct work_struct *work) if (atomic_read(&port->status) & ZFCP_STATUS_PORT_LINK_TEST) goto out; - atomic_set_mask(ZFCP_STATUS_PORT_LINK_TEST, &port->status); + atomic_or(ZFCP_STATUS_PORT_LINK_TEST, &port->status); retval = zfcp_fc_adisc(port); if (retval == 0) return; /* send of ADISC was not possible */ - atomic_clear_mask(ZFCP_STATUS_PORT_LINK_TEST, &port->status); + atomic_andnot(ZFCP_STATUS_PORT_LINK_TEST, &port->status); zfcp_erp_port_forced_reopen(port, 0, "fcltwk1"); out: @@ -640,7 +640,7 @@ static void zfcp_fc_validate_port(struct zfcp_port *port, struct list_head *lh) if (!(atomic_read(&port->status) & ZFCP_STATUS_COMMON_NOESC)) return; - atomic_clear_mask(ZFCP_STATUS_COMMON_NOESC, &port->status); + atomic_andnot(ZFCP_STATUS_COMMON_NOESC, &port->status); if ((port->supported_classes != 0) || !list_empty(&port->unit_list)) diff --git a/drivers/s390/scsi/zfcp_fsf.c b/drivers/s390/scsi/zfcp_fsf.c index 21ec5e2f584c..27b976aa1818 100644 --- a/drivers/s390/scsi/zfcp_fsf.c +++ b/drivers/s390/scsi/zfcp_fsf.c @@ -114,7 +114,7 @@ static void zfcp_fsf_link_down_info_eval(struct zfcp_fsf_req *req, if (atomic_read(&adapter->status) & ZFCP_STATUS_ADAPTER_LINK_UNPLUGGED) return; - atomic_set_mask(ZFCP_STATUS_ADAPTER_LINK_UNPLUGGED, &adapter->status); + atomic_or(ZFCP_STATUS_ADAPTER_LINK_UNPLUGGED, &adapter->status); zfcp_scsi_schedule_rports_block(adapter); @@ -345,7 +345,7 @@ static void zfcp_fsf_protstatus_eval(struct zfcp_fsf_req *req) zfcp_erp_adapter_shutdown(adapter, 0, "fspse_3"); break; case FSF_PROT_HOST_CONNECTION_INITIALIZING: - atomic_set_mask(ZFCP_STATUS_ADAPTER_HOST_CON_INIT, + atomic_or(ZFCP_STATUS_ADAPTER_HOST_CON_INIT, &adapter->status); break; case FSF_PROT_DUPLICATE_REQUEST_ID: @@ -554,7 +554,7 @@ static void zfcp_fsf_exchange_config_data_handler(struct zfcp_fsf_req *req) zfcp_erp_adapter_shutdown(adapter, 0, "fsecdh1"); return; } - atomic_set_mask(ZFCP_STATUS_ADAPTER_XCONFIG_OK, + atomic_or(ZFCP_STATUS_ADAPTER_XCONFIG_OK, &adapter->status); break; case FSF_EXCHANGE_CONFIG_DATA_INCOMPLETE: @@ -567,7 +567,7 @@ static void zfcp_fsf_exchange_config_data_handler(struct zfcp_fsf_req *req) /* avoids adapter shutdown to be able to recognize * events such as LINK UP */ - atomic_set_mask(ZFCP_STATUS_ADAPTER_XCONFIG_OK, + atomic_or(ZFCP_STATUS_ADAPTER_XCONFIG_OK, &adapter->status); zfcp_fsf_link_down_info_eval(req, &qtcb->header.fsf_status_qual.link_down_info); @@ -1394,9 +1394,9 @@ static void zfcp_fsf_open_port_handler(struct zfcp_fsf_req *req) break; case FSF_GOOD: port->handle = header->port_handle; - atomic_set_mask(ZFCP_STATUS_COMMON_OPEN | + atomic_or(ZFCP_STATUS_COMMON_OPEN | ZFCP_STATUS_PORT_PHYS_OPEN, &port->status); - atomic_clear_mask(ZFCP_STATUS_COMMON_ACCESS_BOXED, + atomic_andnot(ZFCP_STATUS_COMMON_ACCESS_BOXED, &port->status); /* check whether D_ID has changed during open */ /* @@ -1677,10 +1677,10 @@ static void zfcp_fsf_close_physical_port_handler(struct zfcp_fsf_req *req) case FSF_PORT_BOXED: /* can't use generic zfcp_erp_modify_port_status because * ZFCP_STATUS_COMMON_OPEN must not be reset for the port */ - atomic_clear_mask(ZFCP_STATUS_PORT_PHYS_OPEN, &port->status); + atomic_andnot(ZFCP_STATUS_PORT_PHYS_OPEN, &port->status); shost_for_each_device(sdev, port->adapter->scsi_host) if (sdev_to_zfcp(sdev)->port == port) - atomic_clear_mask(ZFCP_STATUS_COMMON_OPEN, + atomic_andnot(ZFCP_STATUS_COMMON_OPEN, &sdev_to_zfcp(sdev)->status); zfcp_erp_set_port_status(port, ZFCP_STATUS_COMMON_ACCESS_BOXED); zfcp_erp_port_reopen(port, ZFCP_STATUS_COMMON_ERP_FAILED, @@ -1700,10 +1700,10 @@ static void zfcp_fsf_close_physical_port_handler(struct zfcp_fsf_req *req) /* can't use generic zfcp_erp_modify_port_status because * ZFCP_STATUS_COMMON_OPEN must not be reset for the port */ - atomic_clear_mask(ZFCP_STATUS_PORT_PHYS_OPEN, &port->status); + atomic_andnot(ZFCP_STATUS_PORT_PHYS_OPEN, &port->status); shost_for_each_device(sdev, port->adapter->scsi_host) if (sdev_to_zfcp(sdev)->port == port) - atomic_clear_mask(ZFCP_STATUS_COMMON_OPEN, + atomic_andnot(ZFCP_STATUS_COMMON_OPEN, &sdev_to_zfcp(sdev)->status); break; } @@ -1766,7 +1766,7 @@ static void zfcp_fsf_open_lun_handler(struct zfcp_fsf_req *req) zfcp_sdev = sdev_to_zfcp(sdev); - atomic_clear_mask(ZFCP_STATUS_COMMON_ACCESS_DENIED | + atomic_andnot(ZFCP_STATUS_COMMON_ACCESS_DENIED | ZFCP_STATUS_COMMON_ACCESS_BOXED, &zfcp_sdev->status); @@ -1822,7 +1822,7 @@ static void zfcp_fsf_open_lun_handler(struct zfcp_fsf_req *req) case FSF_GOOD: zfcp_sdev->lun_handle = header->lun_handle; - atomic_set_mask(ZFCP_STATUS_COMMON_OPEN, &zfcp_sdev->status); + atomic_or(ZFCP_STATUS_COMMON_OPEN, &zfcp_sdev->status); break; } } @@ -1913,7 +1913,7 @@ static void zfcp_fsf_close_lun_handler(struct zfcp_fsf_req *req) } break; case FSF_GOOD: - atomic_clear_mask(ZFCP_STATUS_COMMON_OPEN, &zfcp_sdev->status); + atomic_andnot(ZFCP_STATUS_COMMON_OPEN, &zfcp_sdev->status); break; } } diff --git a/drivers/s390/scsi/zfcp_qdio.c b/drivers/s390/scsi/zfcp_qdio.c index 495e1cb3afa6..dbf2b54703f7 100644 --- a/drivers/s390/scsi/zfcp_qdio.c +++ b/drivers/s390/scsi/zfcp_qdio.c @@ -349,7 +349,7 @@ void zfcp_qdio_close(struct zfcp_qdio *qdio) /* clear QDIOUP flag, thus do_QDIO is not called during qdio_shutdown */ spin_lock_irq(&qdio->req_q_lock); - atomic_clear_mask(ZFCP_STATUS_ADAPTER_QDIOUP, &adapter->status); + atomic_andnot(ZFCP_STATUS_ADAPTER_QDIOUP, &adapter->status); spin_unlock_irq(&qdio->req_q_lock); wake_up(&qdio->req_q_wq); @@ -384,7 +384,7 @@ int zfcp_qdio_open(struct zfcp_qdio *qdio) if (atomic_read(&adapter->status) & ZFCP_STATUS_ADAPTER_QDIOUP) return -EIO; - atomic_clear_mask(ZFCP_STATUS_ADAPTER_SIOSL_ISSUED, + atomic_andnot(ZFCP_STATUS_ADAPTER_SIOSL_ISSUED, &qdio->adapter->status); zfcp_qdio_setup_init_data(&init_data, qdio); @@ -396,14 +396,14 @@ int zfcp_qdio_open(struct zfcp_qdio *qdio) goto failed_qdio; if (ssqd.qdioac2 & CHSC_AC2_DATA_DIV_ENABLED) - atomic_set_mask(ZFCP_STATUS_ADAPTER_DATA_DIV_ENABLED, + atomic_or(ZFCP_STATUS_ADAPTER_DATA_DIV_ENABLED, &qdio->adapter->status); if (ssqd.qdioac2 & CHSC_AC2_MULTI_BUFFER_ENABLED) { - atomic_set_mask(ZFCP_STATUS_ADAPTER_MB_ACT, &adapter->status); + atomic_or(ZFCP_STATUS_ADAPTER_MB_ACT, &adapter->status); qdio->max_sbale_per_sbal = QDIO_MAX_ELEMENTS_PER_BUFFER; } else { - atomic_clear_mask(ZFCP_STATUS_ADAPTER_MB_ACT, &adapter->status); + atomic_andnot(ZFCP_STATUS_ADAPTER_MB_ACT, &adapter->status); qdio->max_sbale_per_sbal = QDIO_MAX_ELEMENTS_PER_BUFFER - 1; } @@ -427,7 +427,7 @@ int zfcp_qdio_open(struct zfcp_qdio *qdio) /* set index of first available SBALS / number of available SBALS */ qdio->req_q_idx = 0; atomic_set(&qdio->req_q_free, QDIO_MAX_BUFFERS_PER_Q); - atomic_set_mask(ZFCP_STATUS_ADAPTER_QDIOUP, &qdio->adapter->status); + atomic_or(ZFCP_STATUS_ADAPTER_QDIOUP, &qdio->adapter->status); if (adapter->scsi_host) { adapter->scsi_host->sg_tablesize = qdio->max_sbale_per_req; @@ -499,6 +499,6 @@ void zfcp_qdio_siosl(struct zfcp_adapter *adapter) rc = ccw_device_siosl(adapter->ccw_device); if (!rc) - atomic_set_mask(ZFCP_STATUS_ADAPTER_SIOSL_ISSUED, + atomic_or(ZFCP_STATUS_ADAPTER_SIOSL_ISSUED, &adapter->status); } -- cgit v1.2.3 From 76695af20c015206cffb84b15912be6797d0cca2 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Sun, 2 Aug 2015 17:11:04 +0200 Subject: locking, arch: use WRITE_ONCE()/READ_ONCE() in smp_store_release()/smp_load_acquire() Replace ACCESS_ONCE() macro in smp_store_release() and smp_load_acquire() with WRITE_ONCE() and READ_ONCE() on x86, arm, arm64, ia64, metag, mips, powerpc, s390, sparc and asm-generic since ACCESS_ONCE() does not work reliably on non-scalar types. WRITE_ONCE() and READ_ONCE() were introduced in the following commits: 230fa253df63 ("kernel: Provide READ_ONCE and ASSIGN_ONCE") 43239cbe79fc ("kernel: Change ASSIGN_ONCE(val, x) to WRITE_ONCE(x, val)") Signed-off-by: Andrey Konovalov Signed-off-by: Peter Zijlstra (Intel) Acked-by: Davidlohr Bueso Acked-by: Michael Ellerman (powerpc) Acked-by: Ralf Baechle Cc: Alexander Duyck Cc: Andre Przywara Cc: Arnd Bergmann Cc: Benjamin Herrenschmidt Cc: Borislav Petkov Cc: Catalin Marinas Cc: Christian Borntraeger Cc: David S. Miller Cc: Davidlohr Bueso Cc: Dmitry Vyukov Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Heiko Carstens Cc: James Hogan Cc: Linus Torvalds Cc: Martin Schwidefsky Cc: Paul E. McKenney Cc: Paul Mackerras Cc: Peter Zijlstra Cc: Russell King Cc: Thomas Gleixner Cc: Tony Luck Cc: Will Deacon Cc: linux-arch@vger.kernel.org Link: http://lkml.kernel.org/r/1438528264-714-1-git-send-email-andreyknvl@google.com Signed-off-by: Ingo Molnar --- arch/arm/include/asm/barrier.h | 4 ++-- arch/arm64/include/asm/barrier.h | 4 ++-- arch/ia64/include/asm/barrier.h | 4 ++-- arch/metag/include/asm/barrier.h | 4 ++-- arch/mips/include/asm/barrier.h | 4 ++-- arch/powerpc/include/asm/barrier.h | 4 ++-- arch/s390/include/asm/barrier.h | 4 ++-- arch/sparc/include/asm/barrier_64.h | 4 ++-- arch/x86/include/asm/barrier.h | 8 ++++---- include/asm-generic/barrier.h | 4 ++-- 10 files changed, 22 insertions(+), 22 deletions(-) (limited to 'arch') diff --git a/arch/arm/include/asm/barrier.h b/arch/arm/include/asm/barrier.h index 6c2327e1c732..70393574e0fa 100644 --- a/arch/arm/include/asm/barrier.h +++ b/arch/arm/include/asm/barrier.h @@ -67,12 +67,12 @@ do { \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ ___p1; \ diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h index 0fa47c4275cb..ef93b20bc964 100644 --- a/arch/arm64/include/asm/barrier.h +++ b/arch/arm64/include/asm/barrier.h @@ -44,12 +44,12 @@ do { \ compiletime_assert_atomic_type(*p); \ barrier(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ barrier(); \ ___p1; \ diff --git a/arch/ia64/include/asm/barrier.h b/arch/ia64/include/asm/barrier.h index 843ba435e43b..df896a1c41d3 100644 --- a/arch/ia64/include/asm/barrier.h +++ b/arch/ia64/include/asm/barrier.h @@ -66,12 +66,12 @@ do { \ compiletime_assert_atomic_type(*p); \ barrier(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ barrier(); \ ___p1; \ diff --git a/arch/metag/include/asm/barrier.h b/arch/metag/include/asm/barrier.h index 5a696e507930..172b7e5efc53 100644 --- a/arch/metag/include/asm/barrier.h +++ b/arch/metag/include/asm/barrier.h @@ -90,12 +90,12 @@ static inline void fence(void) do { \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ ___p1; \ diff --git a/arch/mips/include/asm/barrier.h b/arch/mips/include/asm/barrier.h index 7ecba84656d4..752e0b86c171 100644 --- a/arch/mips/include/asm/barrier.h +++ b/arch/mips/include/asm/barrier.h @@ -133,12 +133,12 @@ do { \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ ___p1; \ diff --git a/arch/powerpc/include/asm/barrier.h b/arch/powerpc/include/asm/barrier.h index 51ccc7232042..0eca6efc0631 100644 --- a/arch/powerpc/include/asm/barrier.h +++ b/arch/powerpc/include/asm/barrier.h @@ -76,12 +76,12 @@ do { \ compiletime_assert_atomic_type(*p); \ smp_lwsync(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ smp_lwsync(); \ ___p1; \ diff --git a/arch/s390/include/asm/barrier.h b/arch/s390/include/asm/barrier.h index e6f8615a11eb..d48fe0162331 100644 --- a/arch/s390/include/asm/barrier.h +++ b/arch/s390/include/asm/barrier.h @@ -42,12 +42,12 @@ do { \ compiletime_assert_atomic_type(*p); \ barrier(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ barrier(); \ ___p1; \ diff --git a/arch/sparc/include/asm/barrier_64.h b/arch/sparc/include/asm/barrier_64.h index 809941e33e12..14a928601657 100644 --- a/arch/sparc/include/asm/barrier_64.h +++ b/arch/sparc/include/asm/barrier_64.h @@ -60,12 +60,12 @@ do { __asm__ __volatile__("ba,pt %%xcc, 1f\n\t" \ do { \ compiletime_assert_atomic_type(*p); \ barrier(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ barrier(); \ ___p1; \ diff --git a/arch/x86/include/asm/barrier.h b/arch/x86/include/asm/barrier.h index e51a8f803f55..d2bcfbed11c3 100644 --- a/arch/x86/include/asm/barrier.h +++ b/arch/x86/include/asm/barrier.h @@ -57,12 +57,12 @@ do { \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ ___p1; \ @@ -74,12 +74,12 @@ do { \ do { \ compiletime_assert_atomic_type(*p); \ barrier(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ barrier(); \ ___p1; \ diff --git a/include/asm-generic/barrier.h b/include/asm-generic/barrier.h index 55e3abc2d027..b42afada1280 100644 --- a/include/asm-generic/barrier.h +++ b/include/asm-generic/barrier.h @@ -108,12 +108,12 @@ do { \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ ___p1; \ -- cgit v1.2.3 From 76b235c6bcb16062d663e2ee96db0b69f2e6bc14 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 24 Jul 2015 14:45:44 +0200 Subject: jump_label: Rename JUMP_LABEL_{EN,DIS}ABLE to JUMP_LABEL_{JMP,NOP} Since we've already stepped away from ENABLE is a JMP and DISABLE is a NOP with the branch_default bits, and are going to make it even worse, rename it to make it all clearer. This way we don't mix multiple levels of logic attributes, but have a plain 'physical' name for what the current instruction patching status of a jump label is. This is a first step in removing the naming confusion that has led to a stream of avoidable bugs such as: a833581e372a ("x86, perf: Fix static_key bug in load_mm_cr4()") Signed-off-by: Peter Zijlstra (Intel) Cc: Andrew Morton Cc: Linus Torvalds Cc: Paul E. McKenney Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: linux-kernel@vger.kernel.org [ Beefed up the changelog. ] Signed-off-by: Ingo Molnar --- arch/arm/kernel/jump_label.c | 2 +- arch/arm64/kernel/jump_label.c | 2 +- arch/mips/kernel/jump_label.c | 2 +- arch/powerpc/kernel/jump_label.c | 2 +- arch/s390/kernel/jump_label.c | 2 +- arch/sparc/kernel/jump_label.c | 2 +- arch/x86/kernel/jump_label.c | 2 +- include/linux/jump_label.h | 4 ++-- kernel/jump_label.c | 18 +++++++++--------- 9 files changed, 18 insertions(+), 18 deletions(-) (limited to 'arch') diff --git a/arch/arm/kernel/jump_label.c b/arch/arm/kernel/jump_label.c index e39cbf488cfe..845a5dd9c42b 100644 --- a/arch/arm/kernel/jump_label.c +++ b/arch/arm/kernel/jump_label.c @@ -12,7 +12,7 @@ static void __arch_jump_label_transform(struct jump_entry *entry, void *addr = (void *)entry->code; unsigned int insn; - if (type == JUMP_LABEL_ENABLE) + if (type == JUMP_LABEL_JMP) insn = arm_gen_branch(entry->code, entry->target); else insn = arm_gen_nop(); diff --git a/arch/arm64/kernel/jump_label.c b/arch/arm64/kernel/jump_label.c index 4f1fec7a46db..c2dd1ad3e648 100644 --- a/arch/arm64/kernel/jump_label.c +++ b/arch/arm64/kernel/jump_label.c @@ -28,7 +28,7 @@ void arch_jump_label_transform(struct jump_entry *entry, void *addr = (void *)entry->code; u32 insn; - if (type == JUMP_LABEL_ENABLE) { + if (type == JUMP_LABEL_JMP) { insn = aarch64_insn_gen_branch_imm(entry->code, entry->target, AARCH64_INSN_BRANCH_NOLINK); diff --git a/arch/mips/kernel/jump_label.c b/arch/mips/kernel/jump_label.c index dda800e9e731..3e586daa3a32 100644 --- a/arch/mips/kernel/jump_label.c +++ b/arch/mips/kernel/jump_label.c @@ -51,7 +51,7 @@ void arch_jump_label_transform(struct jump_entry *e, /* Target must have the right alignment and ISA must be preserved. */ BUG_ON((e->target & J_ALIGN_MASK) != J_ISA_BIT); - if (type == JUMP_LABEL_ENABLE) { + if (type == JUMP_LABEL_JMP) { insn.j_format.opcode = J_ISA_BIT ? mm_j32_op : j_op; insn.j_format.target = e->target >> J_RANGE_SHIFT; } else { diff --git a/arch/powerpc/kernel/jump_label.c b/arch/powerpc/kernel/jump_label.c index a1ed8a8c7cb4..6472472093d0 100644 --- a/arch/powerpc/kernel/jump_label.c +++ b/arch/powerpc/kernel/jump_label.c @@ -17,7 +17,7 @@ void arch_jump_label_transform(struct jump_entry *entry, { u32 *addr = (u32 *)(unsigned long)entry->code; - if (type == JUMP_LABEL_ENABLE) + if (type == JUMP_LABEL_JMP) patch_branch(addr, entry->target, 0); else patch_instruction(addr, PPC_INST_NOP); diff --git a/arch/s390/kernel/jump_label.c b/arch/s390/kernel/jump_label.c index a90299600483..a83d2248fea9 100644 --- a/arch/s390/kernel/jump_label.c +++ b/arch/s390/kernel/jump_label.c @@ -64,7 +64,7 @@ static void __jump_label_transform(struct jump_entry *entry, { struct insn old, new; - if (type == JUMP_LABEL_ENABLE) { + if (type == JUMP_LABEL_JMP) { jump_label_make_nop(entry, &old); jump_label_make_branch(entry, &new); } else { diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c index 48565c11e82a..59bbeff55024 100644 --- a/arch/sparc/kernel/jump_label.c +++ b/arch/sparc/kernel/jump_label.c @@ -16,7 +16,7 @@ void arch_jump_label_transform(struct jump_entry *entry, u32 val; u32 *insn = (u32 *) (unsigned long) entry->code; - if (type == JUMP_LABEL_ENABLE) { + if (type == JUMP_LABEL_JMP) { s32 off = (s32)entry->target - (s32)entry->code; #ifdef CONFIG_SPARC64 diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c index 26d5a55a2736..e565e0e4d216 100644 --- a/arch/x86/kernel/jump_label.c +++ b/arch/x86/kernel/jump_label.c @@ -45,7 +45,7 @@ static void __jump_label_transform(struct jump_entry *entry, const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP }; const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5]; - if (type == JUMP_LABEL_ENABLE) { + if (type == JUMP_LABEL_JMP) { if (init) { /* * Jump label is enabled for the first time. diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index f4de473f226b..6a8b4fe10ad8 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -86,8 +86,8 @@ struct static_key { #ifndef __ASSEMBLY__ enum jump_label_type { - JUMP_LABEL_DISABLE = 0, - JUMP_LABEL_ENABLE, + JUMP_LABEL_NOP = 0, + JUMP_LABEL_JMP, }; struct module; diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 52ebaca1b9fc..96d8945c8bf3 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -65,9 +65,9 @@ void static_key_slow_inc(struct static_key *key) jump_label_lock(); if (atomic_read(&key->enabled) == 0) { if (!jump_label_get_branch_default(key)) - jump_label_update(key, JUMP_LABEL_ENABLE); + jump_label_update(key, JUMP_LABEL_JMP); else - jump_label_update(key, JUMP_LABEL_DISABLE); + jump_label_update(key, JUMP_LABEL_NOP); } atomic_inc(&key->enabled); jump_label_unlock(); @@ -88,9 +88,9 @@ static void __static_key_slow_dec(struct static_key *key, schedule_delayed_work(work, rate_limit); } else { if (!jump_label_get_branch_default(key)) - jump_label_update(key, JUMP_LABEL_DISABLE); + jump_label_update(key, JUMP_LABEL_NOP); else - jump_label_update(key, JUMP_LABEL_ENABLE); + jump_label_update(key, JUMP_LABEL_JMP); } jump_label_unlock(); } @@ -184,9 +184,9 @@ static enum jump_label_type jump_label_type(struct static_key *key) bool state = static_key_enabled(key); if ((!true_branch && state) || (true_branch && !state)) - return JUMP_LABEL_ENABLE; + return JUMP_LABEL_JMP; - return JUMP_LABEL_DISABLE; + return JUMP_LABEL_NOP; } void __init jump_label_init(void) @@ -276,7 +276,7 @@ void jump_label_apply_nops(struct module *mod) return; for (iter = iter_start; iter < iter_stop; iter++) { - arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE); + arch_jump_label_transform_static(iter, JUMP_LABEL_NOP); } } @@ -318,8 +318,8 @@ static int jump_label_add_module(struct module *mod) jlm->next = key->next; key->next = jlm; - if (jump_label_type(key) == JUMP_LABEL_ENABLE) - __jump_label_update(key, iter, iter_stop, JUMP_LABEL_ENABLE); + if (jump_label_type(key) == JUMP_LABEL_JMP) + __jump_label_update(key, iter, iter_stop, JUMP_LABEL_JMP); } return 0; -- cgit v1.2.3 From 11276d5306b8e5b438a36bbff855fe792d7eaa61 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 24 Jul 2015 15:09:55 +0200 Subject: locking/static_keys: Add a new static_key interface There are various problems and short-comings with the current static_key interface: - static_key_{true,false}() read like a branch depending on the key value, instead of the actual likely/unlikely branch depending on init value. - static_key_{true,false}() are, as stated above, tied to the static_key init values STATIC_KEY_INIT_{TRUE,FALSE}. - we're limited to the 2 (out of 4) possible options that compile to a default NOP because that's what our arch_static_branch() assembly emits. So provide a new static_key interface: DEFINE_STATIC_KEY_TRUE(name); DEFINE_STATIC_KEY_FALSE(name); Which define a key of different types with an initial true/false value. Then allow: static_branch_likely() static_branch_unlikely() to take a key of either type and emit the right instruction for the case. This means adding a second arch_static_branch_jump() assembly helper which emits a JMP per default. In order to determine the right instruction for the right state, encode the branch type in the LSB of jump_entry::key. This is the final step in removing the naming confusion that has led to a stream of avoidable bugs such as: a833581e372a ("x86, perf: Fix static_key bug in load_mm_cr4()") ... but it also allows new static key combinations that will give us performance enhancements in the subsequent patches. Tested-by: Rabin Vincent # arm Signed-off-by: Peter Zijlstra (Intel) Acked-by: Michael Ellerman # ppc Acked-by: Heiko Carstens # s390 Cc: Andrew Morton Cc: Linus Torvalds Cc: Paul E. McKenney Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: linux-kernel@vger.kernel.org Signed-off-by: Ingo Molnar --- arch/arm/include/asm/jump_label.h | 25 ++++-- arch/arm64/include/asm/jump_label.h | 18 +++- arch/mips/include/asm/jump_label.h | 19 ++++- arch/powerpc/include/asm/jump_label.h | 19 ++++- arch/s390/include/asm/jump_label.h | 19 ++++- arch/sparc/include/asm/jump_label.h | 35 ++++++-- arch/x86/include/asm/jump_label.h | 21 ++++- include/linux/jump_label.h | 149 +++++++++++++++++++++++++++++++--- kernel/jump_label.c | 37 +++++++-- 9 files changed, 298 insertions(+), 44 deletions(-) (limited to 'arch') diff --git a/arch/arm/include/asm/jump_label.h b/arch/arm/include/asm/jump_label.h index 5f337dc5c108..34f7b6980d21 100644 --- a/arch/arm/include/asm/jump_label.h +++ b/arch/arm/include/asm/jump_label.h @@ -4,23 +4,32 @@ #ifndef __ASSEMBLY__ #include +#include #define JUMP_LABEL_NOP_SIZE 4 -#ifdef CONFIG_THUMB2_KERNEL -#define JUMP_LABEL_NOP "nop.w" -#else -#define JUMP_LABEL_NOP "nop" -#endif +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:\n\t" + WASM(nop) "\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection\n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) { asm_volatile_goto("1:\n\t" - JUMP_LABEL_NOP "\n\t" + WASM(b) " %l[l_yes]\n\t" ".pushsection __jump_table, \"aw\"\n\t" ".word 1b, %l[l_yes], %c0\n\t" ".popsection\n\t" - : : "i" (key) : : l_yes); + : : "i" (&((char *)key)[branch]) : : l_yes); return false; l_yes: diff --git a/arch/arm64/include/asm/jump_label.h b/arch/arm64/include/asm/jump_label.h index c0e5165c2f76..1b5e0e843c3a 100644 --- a/arch/arm64/include/asm/jump_label.h +++ b/arch/arm64/include/asm/jump_label.h @@ -26,14 +26,28 @@ #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm goto("1: nop\n\t" ".pushsection __jump_table, \"aw\"\n\t" ".align 3\n\t" ".quad 1b, %l[l_yes], %c0\n\t" ".popsection\n\t" - : : "i"(key) : : l_yes); + : : "i"(&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm goto("1: b %l[l_yes]\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 3\n\t" + ".quad 1b, %l[l_yes], %c0\n\t" + ".popsection\n\t" + : : "i"(&((char *)key)[branch]) : : l_yes); return false; l_yes: diff --git a/arch/mips/include/asm/jump_label.h b/arch/mips/include/asm/jump_label.h index 608aa57799c8..e77672539e8e 100644 --- a/arch/mips/include/asm/jump_label.h +++ b/arch/mips/include/asm/jump_label.h @@ -26,14 +26,29 @@ #define NOP_INSN "nop" #endif -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:\t" NOP_INSN "\n\t" "nop\n\t" ".pushsection __jump_table, \"aw\"\n\t" WORD_INSN " 1b, %l[l_yes], %0\n\t" ".popsection\n\t" - : : "i" (key) : : l_yes); + : : "i" (&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:\tj %l[l_yes]\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + WORD_INSN " 1b, %l[l_yes], %0\n\t" + ".popsection\n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + return false; l_yes: return true; diff --git a/arch/powerpc/include/asm/jump_label.h b/arch/powerpc/include/asm/jump_label.h index efbf9a322a23..47e155f15433 100644 --- a/arch/powerpc/include/asm/jump_label.h +++ b/arch/powerpc/include/asm/jump_label.h @@ -18,14 +18,29 @@ #define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG) #define JUMP_LABEL_NOP_SIZE 4 -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:\n\t" "nop\n\t" ".pushsection __jump_table, \"aw\"\n\t" JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t" ".popsection \n\t" - : : "i" (key) : : l_yes); + : : "i" (&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:\n\t" + "b %l[l_yes]\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + return false; l_yes: return true; diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h index 69972b7957ee..7f9fd5e3f1bf 100644 --- a/arch/s390/include/asm/jump_label.h +++ b/arch/s390/include/asm/jump_label.h @@ -12,14 +12,29 @@ * We use a brcl 0,2 instruction for jump labels at compile time so it * can be easily distinguished from a hotpatch generated instruction. */ -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("0: brcl 0,"__stringify(JUMP_LABEL_NOP_OFFSET)"\n" ".pushsection __jump_table, \"aw\"\n" ".balign 8\n" ".quad 0b, %l[label], %0\n" ".popsection\n" - : : "X" (key) : : label); + : : "X" (&((char *)key)[branch]) : : label); + + return false; +label: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("0: brcl 15, %l[label]\n" + ".pushsection __jump_table, \"aw\"\n" + ".balign 8\n" + ".quad 0b, %l[label], %0\n" + ".popsection\n" + : : "X" (&((char *)key)[branch]) : : label); + return false; label: return true; diff --git a/arch/sparc/include/asm/jump_label.h b/arch/sparc/include/asm/jump_label.h index cc9b04a2b11b..62d0354d1727 100644 --- a/arch/sparc/include/asm/jump_label.h +++ b/arch/sparc/include/asm/jump_label.h @@ -7,16 +7,33 @@ #define JUMP_LABEL_NOP_SIZE 4 -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { - asm_volatile_goto("1:\n\t" - "nop\n\t" - "nop\n\t" - ".pushsection __jump_table, \"aw\"\n\t" - ".align 4\n\t" - ".word 1b, %l[l_yes], %c0\n\t" - ".popsection \n\t" - : : "i" (key) : : l_yes); + asm_volatile_goto("1:\n\t" + "nop\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 4\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:\n\t" + "b %l[l_yes]\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 4\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + return false; l_yes: return true; diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h index a4c1cf7e93f8..28d7a857f9d1 100644 --- a/arch/x86/include/asm/jump_label.h +++ b/arch/x86/include/asm/jump_label.h @@ -16,7 +16,7 @@ # define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC #endif -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:" ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t" @@ -24,7 +24,24 @@ static __always_inline bool arch_static_branch(struct static_key *key) _ASM_ALIGN "\n\t" _ASM_PTR "1b, %l[l_yes], %c0 \n\t" ".popsection \n\t" - : : "i" (key) : : l_yes); + : : "i" (&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:" + ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t" + "2:\n\t" + ".pushsection __jump_table, \"aw\" \n\t" + _ASM_ALIGN "\n\t" + _ASM_PTR "1b, %l[l_yes], %c0 \n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + return false; l_yes: return true; diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 65f0ebac63cf..e337a1961933 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -107,12 +107,12 @@ static inline int static_key_count(struct static_key *key) static __always_inline bool static_key_false(struct static_key *key) { - return arch_static_branch(key); + return arch_static_branch(key, false); } static __always_inline bool static_key_true(struct static_key *key) { - return !static_key_false(key); + return !arch_static_branch(key, true); } extern struct jump_entry __start___jump_table[]; @@ -130,12 +130,12 @@ extern void static_key_slow_inc(struct static_key *key); extern void static_key_slow_dec(struct static_key *key); extern void jump_label_apply_nops(struct module *mod); -#define STATIC_KEY_INIT_TRUE ((struct static_key) \ +#define STATIC_KEY_INIT_TRUE \ { .enabled = ATOMIC_INIT(1), \ - .entries = (void *)JUMP_TYPE_TRUE }) -#define STATIC_KEY_INIT_FALSE ((struct static_key) \ + .entries = (void *)JUMP_TYPE_TRUE } +#define STATIC_KEY_INIT_FALSE \ { .enabled = ATOMIC_INIT(0), \ - .entries = (void *)JUMP_TYPE_FALSE }) + .entries = (void *)JUMP_TYPE_FALSE } #else /* !HAVE_JUMP_LABEL */ @@ -183,10 +183,8 @@ static inline int jump_label_apply_nops(struct module *mod) return 0; } -#define STATIC_KEY_INIT_TRUE ((struct static_key) \ - { .enabled = ATOMIC_INIT(1) }) -#define STATIC_KEY_INIT_FALSE ((struct static_key) \ - { .enabled = ATOMIC_INIT(0) }) +#define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) } +#define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) } #endif /* HAVE_JUMP_LABEL */ @@ -218,6 +216,137 @@ static inline void static_key_disable(struct static_key *key) static_key_slow_dec(key); } +/* -------------------------------------------------------------------------- */ + +/* + * Two type wrappers around static_key, such that we can use compile time + * type differentiation to emit the right code. + * + * All the below code is macros in order to play type games. + */ + +struct static_key_true { + struct static_key key; +}; + +struct static_key_false { + struct static_key key; +}; + +#define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, } +#define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, } + +#define DEFINE_STATIC_KEY_TRUE(name) \ + struct static_key_true name = STATIC_KEY_TRUE_INIT + +#define DEFINE_STATIC_KEY_FALSE(name) \ + struct static_key_false name = STATIC_KEY_FALSE_INIT + +#ifdef HAVE_JUMP_LABEL + +/* + * Combine the right initial value (type) with the right branch order + * to generate the desired result. + * + * + * type\branch| likely (1) | unlikely (0) + * -----------+-----------------------+------------------ + * | | + * true (1) | ... | ... + * | NOP | JMP L + * | | 1: ... + * | L: ... | + * | | + * | | L: + * | | jmp 1b + * | | + * -----------+-----------------------+------------------ + * | | + * false (0) | ... | ... + * | JMP L | NOP + * | | 1: ... + * | L: ... | + * | | + * | | L: + * | | jmp 1b + * | | + * -----------+-----------------------+------------------ + * + * The initial value is encoded in the LSB of static_key::entries, + * type: 0 = false, 1 = true. + * + * The branch type is encoded in the LSB of jump_entry::key, + * branch: 0 = unlikely, 1 = likely. + * + * This gives the following logic table: + * + * enabled type branch instuction + * -----------------------------+----------- + * 0 0 0 | NOP + * 0 0 1 | JMP + * 0 1 0 | NOP + * 0 1 1 | JMP + * + * 1 0 0 | JMP + * 1 0 1 | NOP + * 1 1 0 | JMP + * 1 1 1 | NOP + * + * Which gives the following functions: + * + * dynamic: instruction = enabled ^ branch + * static: instruction = type ^ branch + * + * See jump_label_type() / jump_label_init_type(). + */ + +extern bool ____wrong_branch_error(void); + +#define static_branch_likely(x) \ +({ \ + bool branch; \ + if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \ + branch = !arch_static_branch(&(x)->key, true); \ + else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \ + branch = !arch_static_branch_jump(&(x)->key, true); \ + else \ + branch = ____wrong_branch_error(); \ + branch; \ +}) + +#define static_branch_unlikely(x) \ +({ \ + bool branch; \ + if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \ + branch = arch_static_branch_jump(&(x)->key, false); \ + else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \ + branch = arch_static_branch(&(x)->key, false); \ + else \ + branch = ____wrong_branch_error(); \ + branch; \ +}) + +#else /* !HAVE_JUMP_LABEL */ + +#define static_branch_likely(x) likely(static_key_enabled(&(x)->key)) +#define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key)) + +#endif /* HAVE_JUMP_LABEL */ + +/* + * Advanced usage; refcount, branch is enabled when: count != 0 + */ + +#define static_branch_inc(x) static_key_slow_inc(&(x)->key) +#define static_branch_dec(x) static_key_slow_dec(&(x)->key) + +/* + * Normal usage; boolean enable/disable. + */ + +#define static_branch_enable(x) static_key_enable(&(x)->key) +#define static_branch_disable(x) static_key_disable(&(x)->key) + #endif /* _LINUX_JUMP_LABEL_H */ #endif /* __ASSEMBLY__ */ diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 2e7cc1e4b4b5..8fd00d892286 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -165,16 +165,22 @@ static inline bool static_key_type(struct static_key *key) static inline struct static_key *jump_entry_key(struct jump_entry *entry) { - return (struct static_key *)((unsigned long)entry->key); + return (struct static_key *)((unsigned long)entry->key & ~1UL); +} + +static bool jump_entry_branch(struct jump_entry *entry) +{ + return (unsigned long)entry->key & 1UL; } static enum jump_label_type jump_label_type(struct jump_entry *entry) { struct static_key *key = jump_entry_key(entry); bool enabled = static_key_enabled(key); - bool type = static_key_type(key); + bool branch = jump_entry_branch(entry); - return enabled ^ type; + /* See the comment in linux/jump_label.h */ + return enabled ^ branch; } static void __jump_label_update(struct static_key *key, @@ -205,7 +211,10 @@ void __init jump_label_init(void) for (iter = iter_start; iter < iter_stop; iter++) { struct static_key *iterk; - arch_jump_label_transform_static(iter, jump_label_type(iter)); + /* rewrite NOPs */ + if (jump_label_type(iter) == JUMP_LABEL_NOP) + arch_jump_label_transform_static(iter, JUMP_LABEL_NOP); + iterk = jump_entry_key(iter); if (iterk == key) continue; @@ -225,6 +234,16 @@ void __init jump_label_init(void) #ifdef CONFIG_MODULES +static enum jump_label_type jump_label_init_type(struct jump_entry *entry) +{ + struct static_key *key = jump_entry_key(entry); + bool type = static_key_type(key); + bool branch = jump_entry_branch(entry); + + /* See the comment in linux/jump_label.h */ + return type ^ branch; +} + struct static_key_mod { struct static_key_mod *next; struct jump_entry *entries; @@ -276,8 +295,11 @@ void jump_label_apply_nops(struct module *mod) if (iter_start == iter_stop) return; - for (iter = iter_start; iter < iter_stop; iter++) - arch_jump_label_transform_static(iter, JUMP_LABEL_NOP); + for (iter = iter_start; iter < iter_stop; iter++) { + /* Only write NOPs for arch_branch_static(). */ + if (jump_label_init_type(iter) == JUMP_LABEL_NOP) + arch_jump_label_transform_static(iter, JUMP_LABEL_NOP); + } } static int jump_label_add_module(struct module *mod) @@ -318,7 +340,8 @@ static int jump_label_add_module(struct module *mod) jlm->next = key->next; key->next = jlm; - if (jump_label_type(iter) == JUMP_LABEL_JMP) + /* Only update if we've changed from our initial state */ + if (jump_label_type(iter) != jump_label_init_type(iter)) __jump_label_update(key, iter, iter_stop); } -- cgit v1.2.3 From 1987c947d905baa05bee430178e8eef882f36bd4 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 27 Jul 2015 18:32:09 +0200 Subject: locking/static_keys: Add selftest Add a little selftest that validates all combinations. Signed-off-by: Peter Zijlstra (Intel) Cc: Andrew Morton Cc: Linus Torvalds Cc: Paul E. McKenney Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: linux-kernel@vger.kernel.org Signed-off-by: Ingo Molnar --- arch/Kconfig | 6 ++++++ kernel/jump_label.c | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) (limited to 'arch') diff --git a/arch/Kconfig b/arch/Kconfig index 8a8ea7110de8..a71cdbe2a04d 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -71,6 +71,12 @@ config JUMP_LABEL ( On 32-bit x86, the necessary options added to the compiler flags may increase the size of the kernel slightly. ) +config STATIC_KEYS_SELFTEST + bool "Static key selftest" + depends on JUMP_LABEL + help + Boot time self-test of the branch patching code. + config OPTPROBES def_bool y depends on KPROBES && HAVE_OPTPROBES diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 8fd00d892286..f7dd15d537f9 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -482,4 +482,41 @@ static void jump_label_update(struct static_key *key) __jump_label_update(key, entry, stop); } -#endif +#ifdef CONFIG_STATIC_KEYS_SELFTEST +static DEFINE_STATIC_KEY_TRUE(sk_true); +static DEFINE_STATIC_KEY_FALSE(sk_false); + +static __init int jump_label_test(void) +{ + int i; + + for (i = 0; i < 2; i++) { + WARN_ON(static_key_enabled(&sk_true.key) != true); + WARN_ON(static_key_enabled(&sk_false.key) != false); + + WARN_ON(!static_branch_likely(&sk_true)); + WARN_ON(!static_branch_unlikely(&sk_true)); + WARN_ON(static_branch_likely(&sk_false)); + WARN_ON(static_branch_unlikely(&sk_false)); + + static_branch_disable(&sk_true); + static_branch_enable(&sk_false); + + WARN_ON(static_key_enabled(&sk_true.key) == true); + WARN_ON(static_key_enabled(&sk_false.key) == false); + + WARN_ON(static_branch_likely(&sk_true)); + WARN_ON(static_branch_unlikely(&sk_true)); + WARN_ON(!static_branch_likely(&sk_false)); + WARN_ON(!static_branch_unlikely(&sk_false)); + + static_branch_enable(&sk_true); + static_branch_disable(&sk_false); + } + + return 0; +} +late_initcall(jump_label_test); +#endif /* STATIC_KEYS_SELFTEST */ + +#endif /* HAVE_JUMP_LABEL */ -- cgit v1.2.3 From 3bbfafb77a06327fa1bc9f19bc55b5c558475091 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 24 Jul 2015 16:34:32 +0200 Subject: x86, tsc, locking/static_keys: Employ static_branch_likely() Because of the static_key restrictions we had to take an unconditional jump for the most likely case, causing $I bloat. Rewrite to use the new primitives. Signed-off-by: Peter Zijlstra (Intel) Cc: Andrew Morton Cc: Linus Torvalds Cc: Paul E. McKenney Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: linux-kernel@vger.kernel.org Signed-off-by: Ingo Molnar --- arch/x86/kernel/tsc.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) (limited to 'arch') diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 1bb8bab1b3cb..b9cfd462f7e7 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -38,7 +38,7 @@ static int __read_mostly tsc_unstable; erroneous rdtsc usage on !cpu_has_tsc processors */ static int __read_mostly tsc_disabled = -1; -static struct static_key __use_tsc = STATIC_KEY_INIT; +static DEFINE_STATIC_KEY_FALSE(__use_tsc); int tsc_clocksource_reliable; @@ -274,7 +274,12 @@ done: */ u64 native_sched_clock(void) { - u64 tsc_now; + if (static_branch_likely(&__use_tsc)) { + u64 tsc_now = rdtsc(); + + /* return the value in ns */ + return cycles_2_ns(tsc_now); + } /* * Fall back to jiffies if there's no TSC available: @@ -284,16 +289,9 @@ u64 native_sched_clock(void) * very important for it to be as fast as the platform * can achieve it. ) */ - if (!static_key_false(&__use_tsc)) { - /* No locking but a rare wrong value is not a big deal: */ - return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ); - } - - /* read the Time Stamp Counter: */ - tsc_now = rdtsc(); - /* return the value in ns */ - return cycles_2_ns(tsc_now); + /* No locking but a rare wrong value is not a big deal: */ + return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ); } /* We need to define a real function for sched_clock, to override the @@ -1204,7 +1202,7 @@ void __init tsc_init(void) /* now allow native_sched_clock() to use rdtsc */ tsc_disabled = 0; - static_key_slow_inc(&__use_tsc); + static_branch_enable(&__use_tsc); if (!no_sched_irq_time) enable_sched_clock_irqtime(); -- cgit v1.2.3 From ed79e946732e5311934d7f404b3b4e702e45cb97 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 29 Jul 2015 08:31:24 +0200 Subject: s390/uaccess, locking/static_keys: employ static_branch_likely() Use the new static_branch_likely() primitive to make sure that the most likely case is executed without taking an unconditional branch. This wasn't possible with the old jump label primitives. Signed-off-by: Heiko Carstens Signed-off-by: Peter Zijlstra (Intel) Cc: Andrew Morton Cc: Linus Torvalds Cc: Paul E. McKenney Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: linux-kernel@vger.kernel.org Link: http://lkml.kernel.org/r/20150729064600.GB3953@osiris Signed-off-by: Ingo Molnar --- arch/s390/lib/uaccess.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'arch') diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 4614d415bb58..93cb1d09493d 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -15,7 +15,7 @@ #include #include -static struct static_key have_mvcos = STATIC_KEY_INIT_FALSE; +static DEFINE_STATIC_KEY_FALSE(have_mvcos); static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr, unsigned long size) @@ -104,7 +104,7 @@ static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr, unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n) { - if (static_key_false(&have_mvcos)) + if (static_branch_likely(&have_mvcos)) return copy_from_user_mvcos(to, from, n); return copy_from_user_mvcp(to, from, n); } @@ -177,7 +177,7 @@ static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x, unsigned long __copy_to_user(void __user *to, const void *from, unsigned long n) { - if (static_key_false(&have_mvcos)) + if (static_branch_likely(&have_mvcos)) return copy_to_user_mvcos(to, from, n); return copy_to_user_mvcs(to, from, n); } @@ -240,7 +240,7 @@ static inline unsigned long copy_in_user_mvc(void __user *to, const void __user unsigned long __copy_in_user(void __user *to, const void __user *from, unsigned long n) { - if (static_key_false(&have_mvcos)) + if (static_branch_likely(&have_mvcos)) return copy_in_user_mvcos(to, from, n); return copy_in_user_mvc(to, from, n); } @@ -312,7 +312,7 @@ static inline unsigned long clear_user_xc(void __user *to, unsigned long size) unsigned long __clear_user(void __user *to, unsigned long size) { - if (static_key_false(&have_mvcos)) + if (static_branch_likely(&have_mvcos)) return clear_user_mvcos(to, size); return clear_user_xc(to, size); } @@ -386,7 +386,7 @@ early_param("uaccess_primary", parse_uaccess_pt); static int __init uaccess_init(void) { if (!uaccess_primary && test_facility(27)) - static_key_slow_inc(&have_mvcos); + static_branch_enable(&have_mvcos); return 0; } early_initcall(uaccess_init); -- cgit v1.2.3 From 2b2a85a4d3534b8884fcfa5bb52837f0e1c672bc Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 6 Aug 2015 17:54:41 +0100 Subject: locking/qrwlock: Implement queue_write_unlock() using smp_store_release() Since the following commit: 536fa402221f ("compiler: Allow 1- and 2-byte smp_load_acquire() and smp_store_release()") smp_store_release() supports byte accesses, so use that in writer unlock and remove the conditional macro override. Signed-off-by: Will Deacon Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Waiman Long Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: paulmck@linux.vnet.ibm.com Link: http://lkml.kernel.org/r/1438880084-18856-6-git-send-email-will.deacon@arm.com Signed-off-by: Ingo Molnar --- arch/x86/include/asm/qrwlock.h | 10 ---------- include/asm-generic/qrwlock.h | 9 +-------- 2 files changed, 1 insertion(+), 18 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/qrwlock.h b/arch/x86/include/asm/qrwlock.h index a8810bf135ab..c537cbb038a7 100644 --- a/arch/x86/include/asm/qrwlock.h +++ b/arch/x86/include/asm/qrwlock.h @@ -2,16 +2,6 @@ #define _ASM_X86_QRWLOCK_H #include - -#ifndef CONFIG_X86_PPRO_FENCE -#define queued_write_unlock queued_write_unlock -static inline void queued_write_unlock(struct qrwlock *lock) -{ - barrier(); - ACCESS_ONCE(*(u8 *)&lock->cnts) = 0; -} -#endif - #include #endif /* _ASM_X86_QRWLOCK_H */ diff --git a/include/asm-generic/qrwlock.h b/include/asm-generic/qrwlock.h index deb9e8b0eb9e..eb673dde8879 100644 --- a/include/asm-generic/qrwlock.h +++ b/include/asm-generic/qrwlock.h @@ -134,21 +134,14 @@ static inline void queued_read_unlock(struct qrwlock *lock) atomic_sub(_QR_BIAS, &lock->cnts); } -#ifndef queued_write_unlock /** * queued_write_unlock - release write lock of a queue rwlock * @lock : Pointer to queue rwlock structure */ static inline void queued_write_unlock(struct qrwlock *lock) { - /* - * If the writer field is atomic, it can be cleared directly. - * Otherwise, an atomic subtraction will be used to clear it. - */ - smp_mb__before_atomic(); - atomic_sub(_QW_LOCKED, &lock->cnts); + smp_store_release((u8 *)&lock->cnts, 0); } -#endif /* * Remapping rwlock architecture specific functions to the corresponding -- cgit v1.2.3 From 0ca326de7aa9cb253db9c1a3eb3f0487c8dbf912 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 6 Aug 2015 17:54:44 +0100 Subject: locking, ARM, atomics: Define our SMP atomics in terms of _relaxed() operations By defining our SMP atomics in terms of relaxed operations, we gain a small reduction in code size and have acquire/release/fence variants generated automatically by the core code. Signed-off-by: Will Deacon Signed-off-by: Peter Zijlstra (Intel) Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: Waiman.Long@hp.com Cc: paulmck@linux.vnet.ibm.com Link: http://lkml.kernel.org/r/1438880084-18856-9-git-send-email-will.deacon@arm.com Signed-off-by: Ingo Molnar --- arch/arm/include/asm/atomic.h | 37 ++++++++++++++------------------- arch/arm/include/asm/cmpxchg.h | 47 +++++++----------------------------------- 2 files changed, 24 insertions(+), 60 deletions(-) (limited to 'arch') diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h index 82b75a7cb762..fe3ef397f5a4 100644 --- a/arch/arm/include/asm/atomic.h +++ b/arch/arm/include/asm/atomic.h @@ -57,12 +57,11 @@ static inline void atomic_##op(int i, atomic_t *v) \ } \ #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int atomic_##op##_return_relaxed(int i, atomic_t *v) \ { \ unsigned long tmp; \ int result; \ \ - smp_mb(); \ prefetchw(&v->counter); \ \ __asm__ __volatile__("@ atomic_" #op "_return\n" \ @@ -75,17 +74,17 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ : "r" (&v->counter), "Ir" (i) \ : "cc"); \ \ - smp_mb(); \ - \ return result; \ } -static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new) +#define atomic_add_return_relaxed atomic_add_return_relaxed +#define atomic_sub_return_relaxed atomic_sub_return_relaxed + +static inline int atomic_cmpxchg_relaxed(atomic_t *ptr, int old, int new) { int oldval; unsigned long res; - smp_mb(); prefetchw(&ptr->counter); do { @@ -99,10 +98,9 @@ static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new) : "cc"); } while (res); - smp_mb(); - return oldval; } +#define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed static inline int __atomic_add_unless(atomic_t *v, int a, int u) { @@ -297,12 +295,12 @@ static inline void atomic64_##op(long long i, atomic64_t *v) \ } \ #define ATOMIC64_OP_RETURN(op, op1, op2) \ -static inline long long atomic64_##op##_return(long long i, atomic64_t *v) \ +static inline long long \ +atomic64_##op##_return_relaxed(long long i, atomic64_t *v) \ { \ long long result; \ unsigned long tmp; \ \ - smp_mb(); \ prefetchw(&v->counter); \ \ __asm__ __volatile__("@ atomic64_" #op "_return\n" \ @@ -316,8 +314,6 @@ static inline long long atomic64_##op##_return(long long i, atomic64_t *v) \ : "r" (&v->counter), "r" (i) \ : "cc"); \ \ - smp_mb(); \ - \ return result; \ } @@ -328,6 +324,9 @@ static inline long long atomic64_##op##_return(long long i, atomic64_t *v) \ ATOMIC64_OPS(add, adds, adc) ATOMIC64_OPS(sub, subs, sbc) +#define atomic64_add_return_relaxed atomic64_add_return_relaxed +#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed + #define atomic64_andnot atomic64_andnot ATOMIC64_OP(and, and, and) @@ -339,13 +338,12 @@ ATOMIC64_OP(xor, eor, eor) #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP -static inline long long atomic64_cmpxchg(atomic64_t *ptr, long long old, - long long new) +static inline long long +atomic64_cmpxchg_relaxed(atomic64_t *ptr, long long old, long long new) { long long oldval; unsigned long res; - smp_mb(); prefetchw(&ptr->counter); do { @@ -360,17 +358,15 @@ static inline long long atomic64_cmpxchg(atomic64_t *ptr, long long old, : "cc"); } while (res); - smp_mb(); - return oldval; } +#define atomic64_cmpxchg_relaxed atomic64_cmpxchg_relaxed -static inline long long atomic64_xchg(atomic64_t *ptr, long long new) +static inline long long atomic64_xchg_relaxed(atomic64_t *ptr, long long new) { long long result; unsigned long tmp; - smp_mb(); prefetchw(&ptr->counter); __asm__ __volatile__("@ atomic64_xchg\n" @@ -382,10 +378,9 @@ static inline long long atomic64_xchg(atomic64_t *ptr, long long new) : "r" (&ptr->counter), "r" (new) : "cc"); - smp_mb(); - return result; } +#define atomic64_xchg_relaxed atomic64_xchg_relaxed static inline long long atomic64_dec_if_positive(atomic64_t *v) { diff --git a/arch/arm/include/asm/cmpxchg.h b/arch/arm/include/asm/cmpxchg.h index 1692a05d3207..916a2744d5c6 100644 --- a/arch/arm/include/asm/cmpxchg.h +++ b/arch/arm/include/asm/cmpxchg.h @@ -35,7 +35,6 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size unsigned int tmp; #endif - smp_mb(); prefetchw((const void *)ptr); switch (size) { @@ -98,12 +97,11 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size __bad_xchg(ptr, size), ret = 0; break; } - smp_mb(); return ret; } -#define xchg(ptr, x) ({ \ +#define xchg_relaxed(ptr, x) ({ \ (__typeof__(*(ptr)))__xchg((unsigned long)(x), (ptr), \ sizeof(*(ptr))); \ }) @@ -117,6 +115,8 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size #error "SMP is not supported on this platform" #endif +#define xchg xchg_relaxed + /* * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make * them available. @@ -194,23 +194,11 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, return oldval; } -static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, - unsigned long new, int size) -{ - unsigned long ret; - - smp_mb(); - ret = __cmpxchg(ptr, old, new, size); - smp_mb(); - - return ret; -} - -#define cmpxchg(ptr,o,n) ({ \ - (__typeof__(*(ptr)))__cmpxchg_mb((ptr), \ - (unsigned long)(o), \ - (unsigned long)(n), \ - sizeof(*(ptr))); \ +#define cmpxchg_relaxed(ptr,o,n) ({ \ + (__typeof__(*(ptr)))__cmpxchg((ptr), \ + (unsigned long)(o), \ + (unsigned long)(n), \ + sizeof(*(ptr))); \ }) static inline unsigned long __cmpxchg_local(volatile void *ptr, @@ -273,25 +261,6 @@ static inline unsigned long long __cmpxchg64(unsigned long long *ptr, #define cmpxchg64_local(ptr, o, n) cmpxchg64_relaxed((ptr), (o), (n)) -static inline unsigned long long __cmpxchg64_mb(unsigned long long *ptr, - unsigned long long old, - unsigned long long new) -{ - unsigned long long ret; - - smp_mb(); - ret = __cmpxchg64(ptr, old, new); - smp_mb(); - - return ret; -} - -#define cmpxchg64(ptr, o, n) ({ \ - (__typeof__(*(ptr)))__cmpxchg64_mb((ptr), \ - (unsigned long long)(o), \ - (unsigned long long)(n)); \ -}) - #endif /* __LINUX_ARM_ARCH__ >= 6 */ #endif /* __ASM_ARM_CMPXCHG_H */ -- cgit v1.2.3 From d420acd816c07c7be31bd19d09cbcb16e5572fa6 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 12 Aug 2015 21:04:22 +0200 Subject: jump_label/x86: Work around asm build bug on older/backported GCCs Boris reported that gcc version 4.4.4 20100503 (Red Hat 4.4.4-2) fails to build linux-next kernels that have this fresh commit via the locking tree: 11276d5306b8 ("locking/static_keys: Add a new static_key interface") The problem appears to be that even though @key and @branch are compile time constants, it doesn't see the following expression as an immediate value: &((char *)key)[branch] More recent GCCs don't appear to have this problem. In particular, Red Hat backported the 'asm goto' feature into 4.4, 'normal' 4.4 compilers will not have this feature and thus not run into this asm. The workaround is to supply both values to the asm as immediates and do the addition in asm. Suggested-by: H. Peter Anvin Reported-by: Boris Ostrovsky Tested-by: Boris Ostrovsky Signed-off-by: Peter Zijlstra (Intel) Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Thomas Gleixner Cc: linux-kernel@vger.kernel.org Signed-off-by: Ingo Molnar --- arch/x86/include/asm/jump_label.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'arch') diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h index 28d7a857f9d1..5daeca3d0f9e 100644 --- a/arch/x86/include/asm/jump_label.h +++ b/arch/x86/include/asm/jump_label.h @@ -22,9 +22,9 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool bran ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t" ".pushsection __jump_table, \"aw\" \n\t" _ASM_ALIGN "\n\t" - _ASM_PTR "1b, %l[l_yes], %c0 \n\t" + _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t" ".popsection \n\t" - : : "i" (&((char *)key)[branch]) : : l_yes); + : : "i" (key), "i" (branch) : : l_yes); return false; l_yes: @@ -38,9 +38,9 @@ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool "2:\n\t" ".pushsection __jump_table, \"aw\" \n\t" _ASM_ALIGN "\n\t" - _ASM_PTR "1b, %l[l_yes], %c0 \n\t" + _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t" ".popsection \n\t" - : : "i" (&((char *)key)[branch]) : : l_yes); + : : "i" (key), "i" (branch) : : l_yes); return false; l_yes: -- cgit v1.2.3