From 201af4c0fab02876ef0311e7f7b4083aa138930c Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 27 Dec 2016 14:49:05 -0500 Subject: cgroup: move cgroup files under kernel/cgroup/ They're growing to be too many and planned to get split further. Move them under their own directory. kernel/cgroup.c -> kernel/cgroup/cgroup.c kernel/cgroup_freezer.c -> kernel/cgroup/freezer.c kernel/cgroup_pids.c -> kernel/cgroup/pids.c kernel/cpuset.c -> kernel/cgroup/cpuset.c Signed-off-by: Tejun Heo Acked-by: Acked-by: Zefan Li --- kernel/Makefile | 5 +- kernel/cgroup.c | 6705 ----------------------------------------------- kernel/cgroup/Makefile | 5 + kernel/cgroup/cgroup.c | 6705 +++++++++++++++++++++++++++++++++++++++++++++++ kernel/cgroup/cpuset.c | 2752 +++++++++++++++++++ kernel/cgroup/freezer.c | 481 ++++ kernel/cgroup/pids.c | 348 +++ kernel/cgroup_freezer.c | 481 ---- kernel/cgroup_pids.c | 348 --- kernel/cpuset.c | 2752 ------------------- 10 files changed, 10292 insertions(+), 10290 deletions(-) delete mode 100644 kernel/cgroup.c create mode 100644 kernel/cgroup/Makefile create mode 100644 kernel/cgroup/cgroup.c create mode 100644 kernel/cgroup/cpuset.c create mode 100644 kernel/cgroup/freezer.c create mode 100644 kernel/cgroup/pids.c delete mode 100644 kernel/cgroup_freezer.c delete mode 100644 kernel/cgroup_pids.c delete mode 100644 kernel/cpuset.c (limited to 'kernel') diff --git a/kernel/Makefile b/kernel/Makefile index 12c679f769c6..b302b4731d16 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -64,10 +64,7 @@ obj-$(CONFIG_KEXEC) += kexec.o obj-$(CONFIG_KEXEC_FILE) += kexec_file.o obj-$(CONFIG_BACKTRACE_SELF_TEST) += backtracetest.o obj-$(CONFIG_COMPAT) += compat.o -obj-$(CONFIG_CGROUPS) += cgroup.o -obj-$(CONFIG_CGROUP_FREEZER) += cgroup_freezer.o -obj-$(CONFIG_CGROUP_PIDS) += cgroup_pids.o -obj-$(CONFIG_CPUSETS) += cpuset.o +obj-$(CONFIG_CGROUPS) += cgroup/ obj-$(CONFIG_UTS_NS) += utsname.o obj-$(CONFIG_USER_NS) += user_namespace.o obj-$(CONFIG_PID_NS) += pid_namespace.o diff --git a/kernel/cgroup.c b/kernel/cgroup.c deleted file mode 100644 index 1a815f275849..000000000000 --- a/kernel/cgroup.c +++ /dev/null @@ -1,6705 +0,0 @@ -/* - * Generic process-grouping system. - * - * Based originally on the cpuset system, extracted by Paul Menage - * Copyright (C) 2006 Google, Inc - * - * Notifications support - * Copyright (C) 2009 Nokia Corporation - * Author: Kirill A. Shutemov - * - * Copyright notices from the original cpuset code: - * -------------------------------------------------- - * Copyright (C) 2003 BULL SA. - * Copyright (C) 2004-2006 Silicon Graphics, Inc. - * - * Portions derived from Patrick Mochel's sysfs code. - * sysfs is Copyright (c) 2001-3 Patrick Mochel - * - * 2003-10-10 Written by Simon Derr. - * 2003-10-22 Updates by Stephen Hemminger. - * 2004 May-July Rework by Paul Jackson. - * --------------------------------------------------- - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file COPYING in the main directory of the Linux - * distribution for more details. - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include /* TODO: replace with more sophisticated array */ -#include -#include -#include -#include -#include -#include -#include -#include - -#define CREATE_TRACE_POINTS -#include - -/* - * pidlists linger the following amount before being destroyed. The goal - * is avoiding frequent destruction in the middle of consecutive read calls - * Expiring in the middle is a performance problem not a correctness one. - * 1 sec should be enough. - */ -#define CGROUP_PIDLIST_DESTROY_DELAY HZ - -#define CGROUP_FILE_NAME_MAX (MAX_CGROUP_TYPE_NAMELEN + \ - MAX_CFTYPE_NAME + 2) - -/* - * cgroup_mutex is the master lock. Any modification to cgroup or its - * hierarchy must be performed while holding it. - * - * css_set_lock protects task->cgroups pointer, the list of css_set - * objects, and the chain of tasks off each css_set. - * - * These locks are exported if CONFIG_PROVE_RCU so that accessors in - * cgroup.h can use them for lockdep annotations. - */ -#ifdef CONFIG_PROVE_RCU -DEFINE_MUTEX(cgroup_mutex); -DEFINE_SPINLOCK(css_set_lock); -EXPORT_SYMBOL_GPL(cgroup_mutex); -EXPORT_SYMBOL_GPL(css_set_lock); -#else -static DEFINE_MUTEX(cgroup_mutex); -static DEFINE_SPINLOCK(css_set_lock); -#endif - -/* - * Protects cgroup_idr and css_idr so that IDs can be released without - * grabbing cgroup_mutex. - */ -static DEFINE_SPINLOCK(cgroup_idr_lock); - -/* - * Protects cgroup_file->kn for !self csses. It synchronizes notifications - * against file removal/re-creation across css hiding. - */ -static DEFINE_SPINLOCK(cgroup_file_kn_lock); - -/* - * Protects cgroup_subsys->release_agent_path. Modifying it also requires - * cgroup_mutex. Reading requires either cgroup_mutex or this spinlock. - */ -static DEFINE_SPINLOCK(release_agent_path_lock); - -struct percpu_rw_semaphore cgroup_threadgroup_rwsem; - -#define cgroup_assert_mutex_or_rcu_locked() \ - RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \ - !lockdep_is_held(&cgroup_mutex), \ - "cgroup_mutex or RCU read lock required"); - -/* - * cgroup destruction makes heavy use of work items and there can be a lot - * of concurrent destructions. Use a separate workqueue so that cgroup - * destruction work items don't end up filling up max_active of system_wq - * which may lead to deadlock. - */ -static struct workqueue_struct *cgroup_destroy_wq; - -/* - * pidlist destructions need to be flushed on cgroup destruction. Use a - * separate workqueue as flush domain. - */ -static struct workqueue_struct *cgroup_pidlist_destroy_wq; - -/* generate an array of cgroup subsystem pointers */ -#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys, -static struct cgroup_subsys *cgroup_subsys[] = { -#include -}; -#undef SUBSYS - -/* array of cgroup subsystem names */ -#define SUBSYS(_x) [_x ## _cgrp_id] = #_x, -static const char *cgroup_subsys_name[] = { -#include -}; -#undef SUBSYS - -/* array of static_keys for cgroup_subsys_enabled() and cgroup_subsys_on_dfl() */ -#define SUBSYS(_x) \ - DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_enabled_key); \ - DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_on_dfl_key); \ - EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_enabled_key); \ - EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_on_dfl_key); -#include -#undef SUBSYS - -#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_enabled_key, -static struct static_key_true *cgroup_subsys_enabled_key[] = { -#include -}; -#undef SUBSYS - -#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_on_dfl_key, -static struct static_key_true *cgroup_subsys_on_dfl_key[] = { -#include -}; -#undef SUBSYS - -/* - * The default hierarchy, reserved for the subsystems that are otherwise - * unattached - it never has more than a single cgroup, and all tasks are - * part of that cgroup. - */ -struct cgroup_root cgrp_dfl_root; -EXPORT_SYMBOL_GPL(cgrp_dfl_root); - -/* - * The default hierarchy always exists but is hidden until mounted for the - * first time. This is for backward compatibility. - */ -static bool cgrp_dfl_visible; - -/* Controllers blocked by the commandline in v1 */ -static u16 cgroup_no_v1_mask; - -/* some controllers are not supported in the default hierarchy */ -static u16 cgrp_dfl_inhibit_ss_mask; - -/* some controllers are implicitly enabled on the default hierarchy */ -static unsigned long cgrp_dfl_implicit_ss_mask; - -/* The list of hierarchy roots */ - -static LIST_HEAD(cgroup_roots); -static int cgroup_root_count; - -/* hierarchy ID allocation and mapping, protected by cgroup_mutex */ -static DEFINE_IDR(cgroup_hierarchy_idr); - -/* - * Assign a monotonically increasing serial number to csses. It guarantees - * cgroups with bigger numbers are newer than those with smaller numbers. - * Also, as csses are always appended to the parent's ->children list, it - * guarantees that sibling csses are always sorted in the ascending serial - * number order on the list. Protected by cgroup_mutex. - */ -static u64 css_serial_nr_next = 1; - -/* - * These bitmask flags indicate whether tasks in the fork and exit paths have - * fork/exit handlers to call. This avoids us having to do extra work in the - * fork/exit path to check which subsystems have fork/exit callbacks. - */ -static u16 have_fork_callback __read_mostly; -static u16 have_exit_callback __read_mostly; -static u16 have_free_callback __read_mostly; - -/* cgroup namespace for init task */ -struct cgroup_namespace init_cgroup_ns = { - .count = { .counter = 2, }, - .user_ns = &init_user_ns, - .ns.ops = &cgroupns_operations, - .ns.inum = PROC_CGROUP_INIT_INO, - .root_cset = &init_css_set, -}; - -/* Ditto for the can_fork callback. */ -static u16 have_canfork_callback __read_mostly; - -static struct file_system_type cgroup2_fs_type; -static struct cftype cgroup_dfl_base_files[]; -static struct cftype cgroup_legacy_base_files[]; - -static int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask); -static void cgroup_lock_and_drain_offline(struct cgroup *cgrp); -static int cgroup_apply_control(struct cgroup *cgrp); -static void cgroup_finalize_control(struct cgroup *cgrp, int ret); -static void css_task_iter_advance(struct css_task_iter *it); -static int cgroup_destroy_locked(struct cgroup *cgrp); -static struct cgroup_subsys_state *css_create(struct cgroup *cgrp, - struct cgroup_subsys *ss); -static void css_release(struct percpu_ref *ref); -static void kill_css(struct cgroup_subsys_state *css); -static int cgroup_addrm_files(struct cgroup_subsys_state *css, - struct cgroup *cgrp, struct cftype cfts[], - bool is_add); - -/** - * cgroup_ssid_enabled - cgroup subsys enabled test by subsys ID - * @ssid: subsys ID of interest - * - * cgroup_subsys_enabled() can only be used with literal subsys names which - * is fine for individual subsystems but unsuitable for cgroup core. This - * is slower static_key_enabled() based test indexed by @ssid. - */ -static bool cgroup_ssid_enabled(int ssid) -{ - if (CGROUP_SUBSYS_COUNT == 0) - return false; - - return static_key_enabled(cgroup_subsys_enabled_key[ssid]); -} - -static bool cgroup_ssid_no_v1(int ssid) -{ - return cgroup_no_v1_mask & (1 << ssid); -} - -/** - * cgroup_on_dfl - test whether a cgroup is on the default hierarchy - * @cgrp: the cgroup of interest - * - * The default hierarchy is the v2 interface of cgroup and this function - * can be used to test whether a cgroup is on the default hierarchy for - * cases where a subsystem should behave differnetly depending on the - * interface version. - * - * The set of behaviors which change on the default hierarchy are still - * being determined and the mount option is prefixed with __DEVEL__. - * - * List of changed behaviors: - * - * - Mount options "noprefix", "xattr", "clone_children", "release_agent" - * and "name" are disallowed. - * - * - When mounting an existing superblock, mount options should match. - * - * - Remount is disallowed. - * - * - rename(2) is disallowed. - * - * - "tasks" is removed. Everything should be at process granularity. Use - * "cgroup.procs" instead. - * - * - "cgroup.procs" is not sorted. pids will be unique unless they got - * recycled inbetween reads. - * - * - "release_agent" and "notify_on_release" are removed. Replacement - * notification mechanism will be implemented. - * - * - "cgroup.clone_children" is removed. - * - * - "cgroup.subtree_populated" is available. Its value is 0 if the cgroup - * and its descendants contain no task; otherwise, 1. The file also - * generates kernfs notification which can be monitored through poll and - * [di]notify when the value of the file changes. - * - * - cpuset: tasks will be kept in empty cpusets when hotplug happens and - * take masks of ancestors with non-empty cpus/mems, instead of being - * moved to an ancestor. - * - * - cpuset: a task can be moved into an empty cpuset, and again it takes - * masks of ancestors. - * - * - memcg: use_hierarchy is on by default and the cgroup file for the flag - * is not created. - * - * - blkcg: blk-throttle becomes properly hierarchical. - * - * - debug: disallowed on the default hierarchy. - */ -static bool cgroup_on_dfl(const struct cgroup *cgrp) -{ - return cgrp->root == &cgrp_dfl_root; -} - -/* IDR wrappers which synchronize using cgroup_idr_lock */ -static int cgroup_idr_alloc(struct idr *idr, void *ptr, int start, int end, - gfp_t gfp_mask) -{ - int ret; - - idr_preload(gfp_mask); - spin_lock_bh(&cgroup_idr_lock); - ret = idr_alloc(idr, ptr, start, end, gfp_mask & ~__GFP_DIRECT_RECLAIM); - spin_unlock_bh(&cgroup_idr_lock); - idr_preload_end(); - return ret; -} - -static void *cgroup_idr_replace(struct idr *idr, void *ptr, int id) -{ - void *ret; - - spin_lock_bh(&cgroup_idr_lock); - ret = idr_replace(idr, ptr, id); - spin_unlock_bh(&cgroup_idr_lock); - return ret; -} - -static void cgroup_idr_remove(struct idr *idr, int id) -{ - spin_lock_bh(&cgroup_idr_lock); - idr_remove(idr, id); - spin_unlock_bh(&cgroup_idr_lock); -} - -static struct cgroup *cgroup_parent(struct cgroup *cgrp) -{ - struct cgroup_subsys_state *parent_css = cgrp->self.parent; - - if (parent_css) - return container_of(parent_css, struct cgroup, self); - return NULL; -} - -/* subsystems visibly enabled on a cgroup */ -static u16 cgroup_control(struct cgroup *cgrp) -{ - struct cgroup *parent = cgroup_parent(cgrp); - u16 root_ss_mask = cgrp->root->subsys_mask; - - if (parent) - return parent->subtree_control; - - if (cgroup_on_dfl(cgrp)) - root_ss_mask &= ~(cgrp_dfl_inhibit_ss_mask | - cgrp_dfl_implicit_ss_mask); - return root_ss_mask; -} - -/* subsystems enabled on a cgroup */ -static u16 cgroup_ss_mask(struct cgroup *cgrp) -{ - struct cgroup *parent = cgroup_parent(cgrp); - - if (parent) - return parent->subtree_ss_mask; - - return cgrp->root->subsys_mask; -} - -/** - * cgroup_css - obtain a cgroup's css for the specified subsystem - * @cgrp: the cgroup of interest - * @ss: the subsystem of interest (%NULL returns @cgrp->self) - * - * Return @cgrp's css (cgroup_subsys_state) associated with @ss. This - * function must be called either under cgroup_mutex or rcu_read_lock() and - * the caller is responsible for pinning the returned css if it wants to - * keep accessing it outside the said locks. This function may return - * %NULL if @cgrp doesn't have @subsys_id enabled. - */ -static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp, - struct cgroup_subsys *ss) -{ - if (ss) - return rcu_dereference_check(cgrp->subsys[ss->id], - lockdep_is_held(&cgroup_mutex)); - else - return &cgrp->self; -} - -/** - * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem - * @cgrp: the cgroup of interest - * @ss: the subsystem of interest (%NULL returns @cgrp->self) - * - * Similar to cgroup_css() but returns the effective css, which is defined - * as the matching css of the nearest ancestor including self which has @ss - * enabled. If @ss is associated with the hierarchy @cgrp is on, this - * function is guaranteed to return non-NULL css. - */ -static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp, - struct cgroup_subsys *ss) -{ - lockdep_assert_held(&cgroup_mutex); - - if (!ss) - return &cgrp->self; - - /* - * This function is used while updating css associations and thus - * can't test the csses directly. Test ss_mask. - */ - while (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) { - cgrp = cgroup_parent(cgrp); - if (!cgrp) - return NULL; - } - - return cgroup_css(cgrp, ss); -} - -/** - * cgroup_get_e_css - get a cgroup's effective css for the specified subsystem - * @cgrp: the cgroup of interest - * @ss: the subsystem of interest - * - * Find and get the effective css of @cgrp for @ss. The effective css is - * defined as the matching css of the nearest ancestor including self which - * has @ss enabled. If @ss is not mounted on the hierarchy @cgrp is on, - * the root css is returned, so this function always returns a valid css. - * The returned css must be put using css_put(). - */ -struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgrp, - struct cgroup_subsys *ss) -{ - struct cgroup_subsys_state *css; - - rcu_read_lock(); - - do { - css = cgroup_css(cgrp, ss); - - if (css && css_tryget_online(css)) - goto out_unlock; - cgrp = cgroup_parent(cgrp); - } while (cgrp); - - css = init_css_set.subsys[ss->id]; - css_get(css); -out_unlock: - rcu_read_unlock(); - return css; -} - -/* convenient tests for these bits */ -static inline bool cgroup_is_dead(const struct cgroup *cgrp) -{ - return !(cgrp->self.flags & CSS_ONLINE); -} - -static void cgroup_get(struct cgroup *cgrp) -{ - WARN_ON_ONCE(cgroup_is_dead(cgrp)); - css_get(&cgrp->self); -} - -static bool cgroup_tryget(struct cgroup *cgrp) -{ - return css_tryget(&cgrp->self); -} - -struct cgroup_subsys_state *of_css(struct kernfs_open_file *of) -{ - struct cgroup *cgrp = of->kn->parent->priv; - struct cftype *cft = of_cft(of); - - /* - * This is open and unprotected implementation of cgroup_css(). - * seq_css() is only called from a kernfs file operation which has - * an active reference on the file. Because all the subsystem - * files are drained before a css is disassociated with a cgroup, - * the matching css from the cgroup's subsys table is guaranteed to - * be and stay valid until the enclosing operation is complete. - */ - if (cft->ss) - return rcu_dereference_raw(cgrp->subsys[cft->ss->id]); - else - return &cgrp->self; -} -EXPORT_SYMBOL_GPL(of_css); - -static int notify_on_release(const struct cgroup *cgrp) -{ - return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags); -} - -/** - * for_each_css - iterate all css's of a cgroup - * @css: the iteration cursor - * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end - * @cgrp: the target cgroup to iterate css's of - * - * Should be called under cgroup_[tree_]mutex. - */ -#define for_each_css(css, ssid, cgrp) \ - for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \ - if (!((css) = rcu_dereference_check( \ - (cgrp)->subsys[(ssid)], \ - lockdep_is_held(&cgroup_mutex)))) { } \ - else - -/** - * for_each_e_css - iterate all effective css's of a cgroup - * @css: the iteration cursor - * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end - * @cgrp: the target cgroup to iterate css's of - * - * Should be called under cgroup_[tree_]mutex. - */ -#define for_each_e_css(css, ssid, cgrp) \ - for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \ - if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \ - ; \ - else - -/** - * for_each_subsys - iterate all enabled cgroup subsystems - * @ss: the iteration cursor - * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end - */ -#define for_each_subsys(ss, ssid) \ - for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT && \ - (((ss) = cgroup_subsys[ssid]) || true); (ssid)++) - -/** - * do_each_subsys_mask - filter for_each_subsys with a bitmask - * @ss: the iteration cursor - * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end - * @ss_mask: the bitmask - * - * The block will only run for cases where the ssid-th bit (1 << ssid) of - * @ss_mask is set. - */ -#define do_each_subsys_mask(ss, ssid, ss_mask) do { \ - unsigned long __ss_mask = (ss_mask); \ - if (!CGROUP_SUBSYS_COUNT) { /* to avoid spurious gcc warning */ \ - (ssid) = 0; \ - break; \ - } \ - for_each_set_bit(ssid, &__ss_mask, CGROUP_SUBSYS_COUNT) { \ - (ss) = cgroup_subsys[ssid]; \ - { - -#define while_each_subsys_mask() \ - } \ - } \ -} while (false) - -/* iterate across the hierarchies */ -#define for_each_root(root) \ - list_for_each_entry((root), &cgroup_roots, root_list) - -/* iterate over child cgrps, lock should be held throughout iteration */ -#define cgroup_for_each_live_child(child, cgrp) \ - list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \ - if (({ lockdep_assert_held(&cgroup_mutex); \ - cgroup_is_dead(child); })) \ - ; \ - else - -/* walk live descendants in preorder */ -#define cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) \ - css_for_each_descendant_pre((d_css), cgroup_css((cgrp), NULL)) \ - if (({ lockdep_assert_held(&cgroup_mutex); \ - (dsct) = (d_css)->cgroup; \ - cgroup_is_dead(dsct); })) \ - ; \ - else - -/* walk live descendants in postorder */ -#define cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) \ - css_for_each_descendant_post((d_css), cgroup_css((cgrp), NULL)) \ - if (({ lockdep_assert_held(&cgroup_mutex); \ - (dsct) = (d_css)->cgroup; \ - cgroup_is_dead(dsct); })) \ - ; \ - else - -static void cgroup_release_agent(struct work_struct *work); -static void check_for_release(struct cgroup *cgrp); - -/* - * A cgroup can be associated with multiple css_sets as different tasks may - * belong to different cgroups on different hierarchies. In the other - * direction, a css_set is naturally associated with multiple cgroups. - * This M:N relationship is represented by the following link structure - * which exists for each association and allows traversing the associations - * from both sides. - */ -struct cgrp_cset_link { - /* the cgroup and css_set this link associates */ - struct cgroup *cgrp; - struct css_set *cset; - - /* list of cgrp_cset_links anchored at cgrp->cset_links */ - struct list_head cset_link; - - /* list of cgrp_cset_links anchored at css_set->cgrp_links */ - struct list_head cgrp_link; -}; - -/* - * The default css_set - used by init and its children prior to any - * hierarchies being mounted. It contains a pointer to the root state - * for each subsystem. Also used to anchor the list of css_sets. Not - * reference-counted, to improve performance when child cgroups - * haven't been created. - */ -struct css_set init_css_set = { - .refcount = ATOMIC_INIT(1), - .tasks = LIST_HEAD_INIT(init_css_set.tasks), - .mg_tasks = LIST_HEAD_INIT(init_css_set.mg_tasks), - .task_iters = LIST_HEAD_INIT(init_css_set.task_iters), - .cgrp_links = LIST_HEAD_INIT(init_css_set.cgrp_links), - .mg_preload_node = LIST_HEAD_INIT(init_css_set.mg_preload_node), - .mg_node = LIST_HEAD_INIT(init_css_set.mg_node), -}; - -static int css_set_count = 1; /* 1 for init_css_set */ - -/** - * css_set_populated - does a css_set contain any tasks? - * @cset: target css_set - */ -static bool css_set_populated(struct css_set *cset) -{ - lockdep_assert_held(&css_set_lock); - - return !list_empty(&cset->tasks) || !list_empty(&cset->mg_tasks); -} - -/** - * cgroup_update_populated - updated populated count of a cgroup - * @cgrp: the target cgroup - * @populated: inc or dec populated count - * - * One of the css_sets associated with @cgrp is either getting its first - * task or losing the last. Update @cgrp->populated_cnt accordingly. The - * count is propagated towards root so that a given cgroup's populated_cnt - * is zero iff the cgroup and all its descendants don't contain any tasks. - * - * @cgrp's interface file "cgroup.populated" is zero if - * @cgrp->populated_cnt is zero and 1 otherwise. When @cgrp->populated_cnt - * changes from or to zero, userland is notified that the content of the - * interface file has changed. This can be used to detect when @cgrp and - * its descendants become populated or empty. - */ -static void cgroup_update_populated(struct cgroup *cgrp, bool populated) -{ - lockdep_assert_held(&css_set_lock); - - do { - bool trigger; - - if (populated) - trigger = !cgrp->populated_cnt++; - else - trigger = !--cgrp->populated_cnt; - - if (!trigger) - break; - - check_for_release(cgrp); - cgroup_file_notify(&cgrp->events_file); - - cgrp = cgroup_parent(cgrp); - } while (cgrp); -} - -/** - * css_set_update_populated - update populated state of a css_set - * @cset: target css_set - * @populated: whether @cset is populated or depopulated - * - * @cset is either getting the first task or losing the last. Update the - * ->populated_cnt of all associated cgroups accordingly. - */ -static void css_set_update_populated(struct css_set *cset, bool populated) -{ - struct cgrp_cset_link *link; - - lockdep_assert_held(&css_set_lock); - - list_for_each_entry(link, &cset->cgrp_links, cgrp_link) - cgroup_update_populated(link->cgrp, populated); -} - -/** - * css_set_move_task - move a task from one css_set to another - * @task: task being moved - * @from_cset: css_set @task currently belongs to (may be NULL) - * @to_cset: new css_set @task is being moved to (may be NULL) - * @use_mg_tasks: move to @to_cset->mg_tasks instead of ->tasks - * - * Move @task from @from_cset to @to_cset. If @task didn't belong to any - * css_set, @from_cset can be NULL. If @task is being disassociated - * instead of moved, @to_cset can be NULL. - * - * This function automatically handles populated_cnt updates and - * css_task_iter adjustments but the caller is responsible for managing - * @from_cset and @to_cset's reference counts. - */ -static void css_set_move_task(struct task_struct *task, - struct css_set *from_cset, struct css_set *to_cset, - bool use_mg_tasks) -{ - lockdep_assert_held(&css_set_lock); - - if (to_cset && !css_set_populated(to_cset)) - css_set_update_populated(to_cset, true); - - if (from_cset) { - struct css_task_iter *it, *pos; - - WARN_ON_ONCE(list_empty(&task->cg_list)); - - /* - * @task is leaving, advance task iterators which are - * pointing to it so that they can resume at the next - * position. Advancing an iterator might remove it from - * the list, use safe walk. See css_task_iter_advance*() - * for details. - */ - list_for_each_entry_safe(it, pos, &from_cset->task_iters, - iters_node) - if (it->task_pos == &task->cg_list) - css_task_iter_advance(it); - - list_del_init(&task->cg_list); - if (!css_set_populated(from_cset)) - css_set_update_populated(from_cset, false); - } else { - WARN_ON_ONCE(!list_empty(&task->cg_list)); - } - - if (to_cset) { - /* - * We are synchronized through cgroup_threadgroup_rwsem - * against PF_EXITING setting such that we can't race - * against cgroup_exit() changing the css_set to - * init_css_set and dropping the old one. - */ - WARN_ON_ONCE(task->flags & PF_EXITING); - - rcu_assign_pointer(task->cgroups, to_cset); - list_add_tail(&task->cg_list, use_mg_tasks ? &to_cset->mg_tasks : - &to_cset->tasks); - } -} - -/* - * hash table for cgroup groups. This improves the performance to find - * an existing css_set. This hash doesn't (currently) take into - * account cgroups in empty hierarchies. - */ -#define CSS_SET_HASH_BITS 7 -static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS); - -static unsigned long css_set_hash(struct cgroup_subsys_state *css[]) -{ - unsigned long key = 0UL; - struct cgroup_subsys *ss; - int i; - - for_each_subsys(ss, i) - key += (unsigned long)css[i]; - key = (key >> 16) ^ key; - - return key; -} - -static void put_css_set_locked(struct css_set *cset) -{ - struct cgrp_cset_link *link, *tmp_link; - struct cgroup_subsys *ss; - int ssid; - - lockdep_assert_held(&css_set_lock); - - if (!atomic_dec_and_test(&cset->refcount)) - return; - - /* This css_set is dead. unlink it and release cgroup and css refs */ - for_each_subsys(ss, ssid) { - list_del(&cset->e_cset_node[ssid]); - css_put(cset->subsys[ssid]); - } - hash_del(&cset->hlist); - css_set_count--; - - list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) { - list_del(&link->cset_link); - list_del(&link->cgrp_link); - if (cgroup_parent(link->cgrp)) - cgroup_put(link->cgrp); - kfree(link); - } - - kfree_rcu(cset, rcu_head); -} - -static void put_css_set(struct css_set *cset) -{ - unsigned long flags; - - /* - * Ensure that the refcount doesn't hit zero while any readers - * can see it. Similar to atomic_dec_and_lock(), but for an - * rwlock - */ - if (atomic_add_unless(&cset->refcount, -1, 1)) - return; - - spin_lock_irqsave(&css_set_lock, flags); - put_css_set_locked(cset); - spin_unlock_irqrestore(&css_set_lock, flags); -} - -/* - * refcounted get/put for css_set objects - */ -static inline void get_css_set(struct css_set *cset) -{ - atomic_inc(&cset->refcount); -} - -/** - * compare_css_sets - helper function for find_existing_css_set(). - * @cset: candidate css_set being tested - * @old_cset: existing css_set for a task - * @new_cgrp: cgroup that's being entered by the task - * @template: desired set of css pointers in css_set (pre-calculated) - * - * Returns true if "cset" matches "old_cset" except for the hierarchy - * which "new_cgrp" belongs to, for which it should match "new_cgrp". - */ -static bool compare_css_sets(struct css_set *cset, - struct css_set *old_cset, - struct cgroup *new_cgrp, - struct cgroup_subsys_state *template[]) -{ - struct list_head *l1, *l2; - - /* - * On the default hierarchy, there can be csets which are - * associated with the same set of cgroups but different csses. - * Let's first ensure that csses match. - */ - if (memcmp(template, cset->subsys, sizeof(cset->subsys))) - return false; - - /* - * Compare cgroup pointers in order to distinguish between - * different cgroups in hierarchies. As different cgroups may - * share the same effective css, this comparison is always - * necessary. - */ - l1 = &cset->cgrp_links; - l2 = &old_cset->cgrp_links; - while (1) { - struct cgrp_cset_link *link1, *link2; - struct cgroup *cgrp1, *cgrp2; - - l1 = l1->next; - l2 = l2->next; - /* See if we reached the end - both lists are equal length. */ - if (l1 == &cset->cgrp_links) { - BUG_ON(l2 != &old_cset->cgrp_links); - break; - } else { - BUG_ON(l2 == &old_cset->cgrp_links); - } - /* Locate the cgroups associated with these links. */ - link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link); - link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link); - cgrp1 = link1->cgrp; - cgrp2 = link2->cgrp; - /* Hierarchies should be linked in the same order. */ - BUG_ON(cgrp1->root != cgrp2->root); - - /* - * If this hierarchy is the hierarchy of the cgroup - * that's changing, then we need to check that this - * css_set points to the new cgroup; if it's any other - * hierarchy, then this css_set should point to the - * same cgroup as the old css_set. - */ - if (cgrp1->root == new_cgrp->root) { - if (cgrp1 != new_cgrp) - return false; - } else { - if (cgrp1 != cgrp2) - return false; - } - } - return true; -} - -/** - * find_existing_css_set - init css array and find the matching css_set - * @old_cset: the css_set that we're using before the cgroup transition - * @cgrp: the cgroup that we're moving into - * @template: out param for the new set of csses, should be clear on entry - */ -static struct css_set *find_existing_css_set(struct css_set *old_cset, - struct cgroup *cgrp, - struct cgroup_subsys_state *template[]) -{ - struct cgroup_root *root = cgrp->root; - struct cgroup_subsys *ss; - struct css_set *cset; - unsigned long key; - int i; - - /* - * Build the set of subsystem state objects that we want to see in the - * new css_set. while subsystems can change globally, the entries here - * won't change, so no need for locking. - */ - for_each_subsys(ss, i) { - if (root->subsys_mask & (1UL << i)) { - /* - * @ss is in this hierarchy, so we want the - * effective css from @cgrp. - */ - template[i] = cgroup_e_css(cgrp, ss); - } else { - /* - * @ss is not in this hierarchy, so we don't want - * to change the css. - */ - template[i] = old_cset->subsys[i]; - } - } - - key = css_set_hash(template); - hash_for_each_possible(css_set_table, cset, hlist, key) { - if (!compare_css_sets(cset, old_cset, cgrp, template)) - continue; - - /* This css_set matches what we need */ - return cset; - } - - /* No existing cgroup group matched */ - return NULL; -} - -static void free_cgrp_cset_links(struct list_head *links_to_free) -{ - struct cgrp_cset_link *link, *tmp_link; - - list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) { - list_del(&link->cset_link); - kfree(link); - } -} - -/** - * allocate_cgrp_cset_links - allocate cgrp_cset_links - * @count: the number of links to allocate - * @tmp_links: list_head the allocated links are put on - * - * Allocate @count cgrp_cset_link structures and chain them on @tmp_links - * through ->cset_link. Returns 0 on success or -errno. - */ -static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links) -{ - struct cgrp_cset_link *link; - int i; - - INIT_LIST_HEAD(tmp_links); - - for (i = 0; i < count; i++) { - link = kzalloc(sizeof(*link), GFP_KERNEL); - if (!link) { - free_cgrp_cset_links(tmp_links); - return -ENOMEM; - } - list_add(&link->cset_link, tmp_links); - } - return 0; -} - -/** - * link_css_set - a helper function to link a css_set to a cgroup - * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links() - * @cset: the css_set to be linked - * @cgrp: the destination cgroup - */ -static void link_css_set(struct list_head *tmp_links, struct css_set *cset, - struct cgroup *cgrp) -{ - struct cgrp_cset_link *link; - - BUG_ON(list_empty(tmp_links)); - - if (cgroup_on_dfl(cgrp)) - cset->dfl_cgrp = cgrp; - - link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link); - link->cset = cset; - link->cgrp = cgrp; - - /* - * Always add links to the tail of the lists so that the lists are - * in choronological order. - */ - list_move_tail(&link->cset_link, &cgrp->cset_links); - list_add_tail(&link->cgrp_link, &cset->cgrp_links); - - if (cgroup_parent(cgrp)) - cgroup_get(cgrp); -} - -/** - * find_css_set - return a new css_set with one cgroup updated - * @old_cset: the baseline css_set - * @cgrp: the cgroup to be updated - * - * Return a new css_set that's equivalent to @old_cset, but with @cgrp - * substituted into the appropriate hierarchy. - */ -static struct css_set *find_css_set(struct css_set *old_cset, - struct cgroup *cgrp) -{ - struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { }; - struct css_set *cset; - struct list_head tmp_links; - struct cgrp_cset_link *link; - struct cgroup_subsys *ss; - unsigned long key; - int ssid; - - lockdep_assert_held(&cgroup_mutex); - - /* First see if we already have a cgroup group that matches - * the desired set */ - spin_lock_irq(&css_set_lock); - cset = find_existing_css_set(old_cset, cgrp, template); - if (cset) - get_css_set(cset); - spin_unlock_irq(&css_set_lock); - - if (cset) - return cset; - - cset = kzalloc(sizeof(*cset), GFP_KERNEL); - if (!cset) - return NULL; - - /* Allocate all the cgrp_cset_link objects that we'll need */ - if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) { - kfree(cset); - return NULL; - } - - atomic_set(&cset->refcount, 1); - INIT_LIST_HEAD(&cset->tasks); - INIT_LIST_HEAD(&cset->mg_tasks); - INIT_LIST_HEAD(&cset->task_iters); - INIT_HLIST_NODE(&cset->hlist); - INIT_LIST_HEAD(&cset->cgrp_links); - INIT_LIST_HEAD(&cset->mg_preload_node); - INIT_LIST_HEAD(&cset->mg_node); - - /* Copy the set of subsystem state objects generated in - * find_existing_css_set() */ - memcpy(cset->subsys, template, sizeof(cset->subsys)); - - spin_lock_irq(&css_set_lock); - /* Add reference counts and links from the new css_set. */ - list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) { - struct cgroup *c = link->cgrp; - - if (c->root == cgrp->root) - c = cgrp; - link_css_set(&tmp_links, cset, c); - } - - BUG_ON(!list_empty(&tmp_links)); - - css_set_count++; - - /* Add @cset to the hash table */ - key = css_set_hash(cset->subsys); - hash_add(css_set_table, &cset->hlist, key); - - for_each_subsys(ss, ssid) { - struct cgroup_subsys_state *css = cset->subsys[ssid]; - - list_add_tail(&cset->e_cset_node[ssid], - &css->cgroup->e_csets[ssid]); - css_get(css); - } - - spin_unlock_irq(&css_set_lock); - - return cset; -} - -static struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root) -{ - struct cgroup *root_cgrp = kf_root->kn->priv; - - return root_cgrp->root; -} - -static int cgroup_init_root_id(struct cgroup_root *root) -{ - int id; - - lockdep_assert_held(&cgroup_mutex); - - id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL); - if (id < 0) - return id; - - root->hierarchy_id = id; - return 0; -} - -static void cgroup_exit_root_id(struct cgroup_root *root) -{ - lockdep_assert_held(&cgroup_mutex); - - idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id); -} - -static void cgroup_free_root(struct cgroup_root *root) -{ - if (root) { - idr_destroy(&root->cgroup_idr); - kfree(root); - } -} - -static void cgroup_destroy_root(struct cgroup_root *root) -{ - struct cgroup *cgrp = &root->cgrp; - struct cgrp_cset_link *link, *tmp_link; - - trace_cgroup_destroy_root(root); - - cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); - - BUG_ON(atomic_read(&root->nr_cgrps)); - BUG_ON(!list_empty(&cgrp->self.children)); - - /* Rebind all subsystems back to the default hierarchy */ - WARN_ON(rebind_subsystems(&cgrp_dfl_root, root->subsys_mask)); - - /* - * Release all the links from cset_links to this hierarchy's - * root cgroup - */ - spin_lock_irq(&css_set_lock); - - list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) { - list_del(&link->cset_link); - list_del(&link->cgrp_link); - kfree(link); - } - - spin_unlock_irq(&css_set_lock); - - if (!list_empty(&root->root_list)) { - list_del(&root->root_list); - cgroup_root_count--; - } - - cgroup_exit_root_id(root); - - mutex_unlock(&cgroup_mutex); - - kernfs_destroy_root(root->kf_root); - cgroup_free_root(root); -} - -/* - * look up cgroup associated with current task's cgroup namespace on the - * specified hierarchy - */ -static struct cgroup * -current_cgns_cgroup_from_root(struct cgroup_root *root) -{ - struct cgroup *res = NULL; - struct css_set *cset; - - lockdep_assert_held(&css_set_lock); - - rcu_read_lock(); - - cset = current->nsproxy->cgroup_ns->root_cset; - if (cset == &init_css_set) { - res = &root->cgrp; - } else { - struct cgrp_cset_link *link; - - list_for_each_entry(link, &cset->cgrp_links, cgrp_link) { - struct cgroup *c = link->cgrp; - - if (c->root == root) { - res = c; - break; - } - } - } - rcu_read_unlock(); - - BUG_ON(!res); - return res; -} - -/* look up cgroup associated with given css_set on the specified hierarchy */ -static struct cgroup *cset_cgroup_from_root(struct css_set *cset, - struct cgroup_root *root) -{ - struct cgroup *res = NULL; - - lockdep_assert_held(&cgroup_mutex); - lockdep_assert_held(&css_set_lock); - - if (cset == &init_css_set) { - res = &root->cgrp; - } else { - struct cgrp_cset_link *link; - - list_for_each_entry(link, &cset->cgrp_links, cgrp_link) { - struct cgroup *c = link->cgrp; - - if (c->root == root) { - res = c; - break; - } - } - } - - BUG_ON(!res); - return res; -} - -/* - * Return the cgroup for "task" from the given hierarchy. Must be - * called with cgroup_mutex and css_set_lock held. - */ -static struct cgroup *task_cgroup_from_root(struct task_struct *task, - struct cgroup_root *root) -{ - /* - * No need to lock the task - since we hold cgroup_mutex the - * task can't change groups, so the only thing that can happen - * is that it exits and its css is set back to init_css_set. - */ - return cset_cgroup_from_root(task_css_set(task), root); -} - -/* - * A task must hold cgroup_mutex to modify cgroups. - * - * Any task can increment and decrement the count field without lock. - * So in general, code holding cgroup_mutex can't rely on the count - * field not changing. However, if the count goes to zero, then only - * cgroup_attach_task() can increment it again. Because a count of zero - * means that no tasks are currently attached, therefore there is no - * way a task attached to that cgroup can fork (the other way to - * increment the count). So code holding cgroup_mutex can safely - * assume that if the count is zero, it will stay zero. Similarly, if - * a task holds cgroup_mutex on a cgroup with zero count, it - * knows that the cgroup won't be removed, as cgroup_rmdir() - * needs that mutex. - * - * A cgroup can only be deleted if both its 'count' of using tasks - * is zero, and its list of 'children' cgroups is empty. Since all - * tasks in the system use _some_ cgroup, and since there is always at - * least one task in the system (init, pid == 1), therefore, root cgroup - * always has either children cgroups and/or using tasks. So we don't - * need a special hack to ensure that root cgroup cannot be deleted. - * - * P.S. One more locking exception. RCU is used to guard the - * update of a tasks cgroup pointer by cgroup_attach_task() - */ - -static struct kernfs_syscall_ops cgroup_kf_syscall_ops; -static const struct file_operations proc_cgroupstats_operations; - -static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft, - char *buf) -{ - struct cgroup_subsys *ss = cft->ss; - - if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) && - !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) - snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s", - cgroup_on_dfl(cgrp) ? ss->name : ss->legacy_name, - cft->name); - else - strncpy(buf, cft->name, CGROUP_FILE_NAME_MAX); - return buf; -} - -/** - * cgroup_file_mode - deduce file mode of a control file - * @cft: the control file in question - * - * S_IRUGO for read, S_IWUSR for write. - */ -static umode_t cgroup_file_mode(const struct cftype *cft) -{ - umode_t mode = 0; - - if (cft->read_u64 || cft->read_s64 || cft->seq_show) - mode |= S_IRUGO; - - if (cft->write_u64 || cft->write_s64 || cft->write) { - if (cft->flags & CFTYPE_WORLD_WRITABLE) - mode |= S_IWUGO; - else - mode |= S_IWUSR; - } - - return mode; -} - -/** - * cgroup_calc_subtree_ss_mask - calculate subtree_ss_mask - * @subtree_control: the new subtree_control mask to consider - * @this_ss_mask: available subsystems - * - * On the default hierarchy, a subsystem may request other subsystems to be - * enabled together through its ->depends_on mask. In such cases, more - * subsystems than specified in "cgroup.subtree_control" may be enabled. - * - * This function calculates which subsystems need to be enabled if - * @subtree_control is to be applied while restricted to @this_ss_mask. - */ -static u16 cgroup_calc_subtree_ss_mask(u16 subtree_control, u16 this_ss_mask) -{ - u16 cur_ss_mask = subtree_control; - struct cgroup_subsys *ss; - int ssid; - - lockdep_assert_held(&cgroup_mutex); - - cur_ss_mask |= cgrp_dfl_implicit_ss_mask; - - while (true) { - u16 new_ss_mask = cur_ss_mask; - - do_each_subsys_mask(ss, ssid, cur_ss_mask) { - new_ss_mask |= ss->depends_on; - } while_each_subsys_mask(); - - /* - * Mask out subsystems which aren't available. This can - * happen only if some depended-upon subsystems were bound - * to non-default hierarchies. - */ - new_ss_mask &= this_ss_mask; - - if (new_ss_mask == cur_ss_mask) - break; - cur_ss_mask = new_ss_mask; - } - - return cur_ss_mask; -} - -/** - * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods - * @kn: the kernfs_node being serviced - * - * This helper undoes cgroup_kn_lock_live() and should be invoked before - * the method finishes if locking succeeded. Note that once this function - * returns the cgroup returned by cgroup_kn_lock_live() may become - * inaccessible any time. If the caller intends to continue to access the - * cgroup, it should pin it before invoking this function. - */ -static void cgroup_kn_unlock(struct kernfs_node *kn) -{ - struct cgroup *cgrp; - - if (kernfs_type(kn) == KERNFS_DIR) - cgrp = kn->priv; - else - cgrp = kn->parent->priv; - - mutex_unlock(&cgroup_mutex); - - kernfs_unbreak_active_protection(kn); - cgroup_put(cgrp); -} - -/** - * cgroup_kn_lock_live - locking helper for cgroup kernfs methods - * @kn: the kernfs_node being serviced - * @drain_offline: perform offline draining on the cgroup - * - * This helper is to be used by a cgroup kernfs method currently servicing - * @kn. It breaks the active protection, performs cgroup locking and - * verifies that the associated cgroup is alive. Returns the cgroup if - * alive; otherwise, %NULL. A successful return should be undone by a - * matching cgroup_kn_unlock() invocation. If @drain_offline is %true, the - * cgroup is drained of offlining csses before return. - * - * Any cgroup kernfs method implementation which requires locking the - * associated cgroup should use this helper. It avoids nesting cgroup - * locking under kernfs active protection and allows all kernfs operations - * including self-removal. - */ -static struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn, - bool drain_offline) -{ - struct cgroup *cgrp; - - if (kernfs_type(kn) == KERNFS_DIR) - cgrp = kn->priv; - else - cgrp = kn->parent->priv; - - /* - * We're gonna grab cgroup_mutex which nests outside kernfs - * active_ref. cgroup liveliness check alone provides enough - * protection against removal. Ensure @cgrp stays accessible and - * break the active_ref protection. - */ - if (!cgroup_tryget(cgrp)) - return NULL; - kernfs_break_active_protection(kn); - - if (drain_offline) - cgroup_lock_and_drain_offline(cgrp); - else - mutex_lock(&cgroup_mutex); - - if (!cgroup_is_dead(cgrp)) - return cgrp; - - cgroup_kn_unlock(kn); - return NULL; -} - -static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft) -{ - char name[CGROUP_FILE_NAME_MAX]; - - lockdep_assert_held(&cgroup_mutex); - - if (cft->file_offset) { - struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss); - struct cgroup_file *cfile = (void *)css + cft->file_offset; - - spin_lock_irq(&cgroup_file_kn_lock); - cfile->kn = NULL; - spin_unlock_irq(&cgroup_file_kn_lock); - } - - kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name)); -} - -/** - * css_clear_dir - remove subsys files in a cgroup directory - * @css: taget css - */ -static void css_clear_dir(struct cgroup_subsys_state *css) -{ - struct cgroup *cgrp = css->cgroup; - struct cftype *cfts; - - if (!(css->flags & CSS_VISIBLE)) - return; - - css->flags &= ~CSS_VISIBLE; - - list_for_each_entry(cfts, &css->ss->cfts, node) - cgroup_addrm_files(css, cgrp, cfts, false); -} - -/** - * css_populate_dir - create subsys files in a cgroup directory - * @css: target css - * - * On failure, no file is added. - */ -static int css_populate_dir(struct cgroup_subsys_state *css) -{ - struct cgroup *cgrp = css->cgroup; - struct cftype *cfts, *failed_cfts; - int ret; - - if ((css->flags & CSS_VISIBLE) || !cgrp->kn) - return 0; - - if (!css->ss) { - if (cgroup_on_dfl(cgrp)) - cfts = cgroup_dfl_base_files; - else - cfts = cgroup_legacy_base_files; - - return cgroup_addrm_files(&cgrp->self, cgrp, cfts, true); - } - - list_for_each_entry(cfts, &css->ss->cfts, node) { - ret = cgroup_addrm_files(css, cgrp, cfts, true); - if (ret < 0) { - failed_cfts = cfts; - goto err; - } - } - - css->flags |= CSS_VISIBLE; - - return 0; -err: - list_for_each_entry(cfts, &css->ss->cfts, node) { - if (cfts == failed_cfts) - break; - cgroup_addrm_files(css, cgrp, cfts, false); - } - return ret; -} - -static int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask) -{ - struct cgroup *dcgrp = &dst_root->cgrp; - struct cgroup_subsys *ss; - int ssid, i, ret; - - lockdep_assert_held(&cgroup_mutex); - - do_each_subsys_mask(ss, ssid, ss_mask) { - /* - * If @ss has non-root csses attached to it, can't move. - * If @ss is an implicit controller, it is exempt from this - * rule and can be stolen. - */ - if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)) && - !ss->implicit_on_dfl) - return -EBUSY; - - /* can't move between two non-dummy roots either */ - if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root) - return -EBUSY; - } while_each_subsys_mask(); - - do_each_subsys_mask(ss, ssid, ss_mask) { - struct cgroup_root *src_root = ss->root; - struct cgroup *scgrp = &src_root->cgrp; - struct cgroup_subsys_state *css = cgroup_css(scgrp, ss); - struct css_set *cset; - - WARN_ON(!css || cgroup_css(dcgrp, ss)); - - /* disable from the source */ - src_root->subsys_mask &= ~(1 << ssid); - WARN_ON(cgroup_apply_control(scgrp)); - cgroup_finalize_control(scgrp, 0); - - /* rebind */ - RCU_INIT_POINTER(scgrp->subsys[ssid], NULL); - rcu_assign_pointer(dcgrp->subsys[ssid], css); - ss->root = dst_root; - css->cgroup = dcgrp; - - spin_lock_irq(&css_set_lock); - hash_for_each(css_set_table, i, cset, hlist) - list_move_tail(&cset->e_cset_node[ss->id], - &dcgrp->e_csets[ss->id]); - spin_unlock_irq(&css_set_lock); - - /* default hierarchy doesn't enable controllers by default */ - dst_root->subsys_mask |= 1 << ssid; - if (dst_root == &cgrp_dfl_root) { - static_branch_enable(cgroup_subsys_on_dfl_key[ssid]); - } else { - dcgrp->subtree_control |= 1 << ssid; - static_branch_disable(cgroup_subsys_on_dfl_key[ssid]); - } - - ret = cgroup_apply_control(dcgrp); - if (ret) - pr_warn("partial failure to rebind %s controller (err=%d)\n", - ss->name, ret); - - if (ss->bind) - ss->bind(css); - } while_each_subsys_mask(); - - kernfs_activate(dcgrp->kn); - return 0; -} - -static int cgroup_show_path(struct seq_file *sf, struct kernfs_node *kf_node, - struct kernfs_root *kf_root) -{ - int len = 0; - char *buf = NULL; - struct cgroup_root *kf_cgroot = cgroup_root_from_kf(kf_root); - struct cgroup *ns_cgroup; - - buf = kmalloc(PATH_MAX, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - spin_lock_irq(&css_set_lock); - ns_cgroup = current_cgns_cgroup_from_root(kf_cgroot); - len = kernfs_path_from_node(kf_node, ns_cgroup->kn, buf, PATH_MAX); - spin_unlock_irq(&css_set_lock); - - if (len >= PATH_MAX) - len = -ERANGE; - else if (len > 0) { - seq_escape(sf, buf, " \t\n\\"); - len = 0; - } - kfree(buf); - return len; -} - -static int cgroup_show_options(struct seq_file *seq, - struct kernfs_root *kf_root) -{ - struct cgroup_root *root = cgroup_root_from_kf(kf_root); - struct cgroup_subsys *ss; - int ssid; - - if (root != &cgrp_dfl_root) - for_each_subsys(ss, ssid) - if (root->subsys_mask & (1 << ssid)) - seq_show_option(seq, ss->legacy_name, NULL); - if (root->flags & CGRP_ROOT_NOPREFIX) - seq_puts(seq, ",noprefix"); - if (root->flags & CGRP_ROOT_XATTR) - seq_puts(seq, ",xattr"); - - spin_lock(&release_agent_path_lock); - if (strlen(root->release_agent_path)) - seq_show_option(seq, "release_agent", - root->release_agent_path); - spin_unlock(&release_agent_path_lock); - - if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags)) - seq_puts(seq, ",clone_children"); - if (strlen(root->name)) - seq_show_option(seq, "name", root->name); - return 0; -} - -struct cgroup_sb_opts { - u16 subsys_mask; - unsigned int flags; - char *release_agent; - bool cpuset_clone_children; - char *name; - /* User explicitly requested empty subsystem */ - bool none; -}; - -static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts) -{ - char *token, *o = data; - bool all_ss = false, one_ss = false; - u16 mask = U16_MAX; - struct cgroup_subsys *ss; - int nr_opts = 0; - int i; - -#ifdef CONFIG_CPUSETS - mask = ~((u16)1 << cpuset_cgrp_id); -#endif - - memset(opts, 0, sizeof(*opts)); - - while ((token = strsep(&o, ",")) != NULL) { - nr_opts++; - - if (!*token) - return -EINVAL; - if (!strcmp(token, "none")) { - /* Explicitly have no subsystems */ - opts->none = true; - continue; - } - if (!strcmp(token, "all")) { - /* Mutually exclusive option 'all' + subsystem name */ - if (one_ss) - return -EINVAL; - all_ss = true; - continue; - } - if (!strcmp(token, "noprefix")) { - opts->flags |= CGRP_ROOT_NOPREFIX; - continue; - } - if (!strcmp(token, "clone_children")) { - opts->cpuset_clone_children = true; - continue; - } - if (!strcmp(token, "xattr")) { - opts->flags |= CGRP_ROOT_XATTR; - continue; - } - if (!strncmp(token, "release_agent=", 14)) { - /* Specifying two release agents is forbidden */ - if (opts->release_agent) - return -EINVAL; - opts->release_agent = - kstrndup(token + 14, PATH_MAX - 1, GFP_KERNEL); - if (!opts->release_agent) - return -ENOMEM; - continue; - } - if (!strncmp(token, "name=", 5)) { - const char *name = token + 5; - /* Can't specify an empty name */ - if (!strlen(name)) - return -EINVAL; - /* Must match [\w.-]+ */ - for (i = 0; i < strlen(name); i++) { - char c = name[i]; - if (isalnum(c)) - continue; - if ((c == '.') || (c == '-') || (c == '_')) - continue; - return -EINVAL; - } - /* Specifying two names is forbidden */ - if (opts->name) - return -EINVAL; - opts->name = kstrndup(name, - MAX_CGROUP_ROOT_NAMELEN - 1, - GFP_KERNEL); - if (!opts->name) - return -ENOMEM; - - continue; - } - - for_each_subsys(ss, i) { - if (strcmp(token, ss->legacy_name)) - continue; - if (!cgroup_ssid_enabled(i)) - continue; - if (cgroup_ssid_no_v1(i)) - continue; - - /* Mutually exclusive option 'all' + subsystem name */ - if (all_ss) - return -EINVAL; - opts->subsys_mask |= (1 << i); - one_ss = true; - - break; - } - if (i == CGROUP_SUBSYS_COUNT) - return -ENOENT; - } - - /* - * If the 'all' option was specified select all the subsystems, - * otherwise if 'none', 'name=' and a subsystem name options were - * not specified, let's default to 'all' - */ - if (all_ss || (!one_ss && !opts->none && !opts->name)) - for_each_subsys(ss, i) - if (cgroup_ssid_enabled(i) && !cgroup_ssid_no_v1(i)) - opts->subsys_mask |= (1 << i); - - /* - * We either have to specify by name or by subsystems. (So all - * empty hierarchies must have a name). - */ - if (!opts->subsys_mask && !opts->name) - return -EINVAL; - - /* - * Option noprefix was introduced just for backward compatibility - * with the old cpuset, so we allow noprefix only if mounting just - * the cpuset subsystem. - */ - if ((opts->flags & CGRP_ROOT_NOPREFIX) && (opts->subsys_mask & mask)) - return -EINVAL; - - /* Can't specify "none" and some subsystems */ - if (opts->subsys_mask && opts->none) - return -EINVAL; - - return 0; -} - -static int cgroup_remount(struct kernfs_root *kf_root, int *flags, char *data) -{ - int ret = 0; - struct cgroup_root *root = cgroup_root_from_kf(kf_root); - struct cgroup_sb_opts opts; - u16 added_mask, removed_mask; - - if (root == &cgrp_dfl_root) { - pr_err("remount is not allowed\n"); - return -EINVAL; - } - - cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); - - /* See what subsystems are wanted */ - ret = parse_cgroupfs_options(data, &opts); - if (ret) - goto out_unlock; - - if (opts.subsys_mask != root->subsys_mask || opts.release_agent) - pr_warn("option changes via remount are deprecated (pid=%d comm=%s)\n", - task_tgid_nr(current), current->comm); - - added_mask = opts.subsys_mask & ~root->subsys_mask; - removed_mask = root->subsys_mask & ~opts.subsys_mask; - - /* Don't allow flags or name to change at remount */ - if ((opts.flags ^ root->flags) || - (opts.name && strcmp(opts.name, root->name))) { - pr_err("option or name mismatch, new: 0x%x \"%s\", old: 0x%x \"%s\"\n", - opts.flags, opts.name ?: "", root->flags, root->name); - ret = -EINVAL; - goto out_unlock; - } - - /* remounting is not allowed for populated hierarchies */ - if (!list_empty(&root->cgrp.self.children)) { - ret = -EBUSY; - goto out_unlock; - } - - ret = rebind_subsystems(root, added_mask); - if (ret) - goto out_unlock; - - WARN_ON(rebind_subsystems(&cgrp_dfl_root, removed_mask)); - - if (opts.release_agent) { - spin_lock(&release_agent_path_lock); - strcpy(root->release_agent_path, opts.release_agent); - spin_unlock(&release_agent_path_lock); - } - - trace_cgroup_remount(root); - - out_unlock: - kfree(opts.release_agent); - kfree(opts.name); - mutex_unlock(&cgroup_mutex); - return ret; -} - -/* - * To reduce the fork() overhead for systems that are not actually using - * their cgroups capability, we don't maintain the lists running through - * each css_set to its tasks until we see the list actually used - in other - * words after the first mount. - */ -static bool use_task_css_set_links __read_mostly; - -static void cgroup_enable_task_cg_lists(void) -{ - struct task_struct *p, *g; - - spin_lock_irq(&css_set_lock); - - if (use_task_css_set_links) - goto out_unlock; - - use_task_css_set_links = true; - - /* - * We need tasklist_lock because RCU is not safe against - * while_each_thread(). Besides, a forking task that has passed - * cgroup_post_fork() without seeing use_task_css_set_links = 1 - * is not guaranteed to have its child immediately visible in the - * tasklist if we walk through it with RCU. - */ - read_lock(&tasklist_lock); - do_each_thread(g, p) { - WARN_ON_ONCE(!list_empty(&p->cg_list) || - task_css_set(p) != &init_css_set); - - /* - * We should check if the process is exiting, otherwise - * it will race with cgroup_exit() in that the list - * entry won't be deleted though the process has exited. - * Do it while holding siglock so that we don't end up - * racing against cgroup_exit(). - * - * Interrupts were already disabled while acquiring - * the css_set_lock, so we do not need to disable it - * again when acquiring the sighand->siglock here. - */ - spin_lock(&p->sighand->siglock); - if (!(p->flags & PF_EXITING)) { - struct css_set *cset = task_css_set(p); - - if (!css_set_populated(cset)) - css_set_update_populated(cset, true); - list_add_tail(&p->cg_list, &cset->tasks); - get_css_set(cset); - } - spin_unlock(&p->sighand->siglock); - } while_each_thread(g, p); - read_unlock(&tasklist_lock); -out_unlock: - spin_unlock_irq(&css_set_lock); -} - -static void init_cgroup_housekeeping(struct cgroup *cgrp) -{ - struct cgroup_subsys *ss; - int ssid; - - INIT_LIST_HEAD(&cgrp->self.sibling); - INIT_LIST_HEAD(&cgrp->self.children); - INIT_LIST_HEAD(&cgrp->cset_links); - INIT_LIST_HEAD(&cgrp->pidlists); - mutex_init(&cgrp->pidlist_mutex); - cgrp->self.cgroup = cgrp; - cgrp->self.flags |= CSS_ONLINE; - - for_each_subsys(ss, ssid) - INIT_LIST_HEAD(&cgrp->e_csets[ssid]); - - init_waitqueue_head(&cgrp->offline_waitq); - INIT_WORK(&cgrp->release_agent_work, cgroup_release_agent); -} - -static void init_cgroup_root(struct cgroup_root *root, - struct cgroup_sb_opts *opts) -{ - struct cgroup *cgrp = &root->cgrp; - - INIT_LIST_HEAD(&root->root_list); - atomic_set(&root->nr_cgrps, 1); - cgrp->root = root; - init_cgroup_housekeeping(cgrp); - idr_init(&root->cgroup_idr); - - root->flags = opts->flags; - if (opts->release_agent) - strcpy(root->release_agent_path, opts->release_agent); - if (opts->name) - strcpy(root->name, opts->name); - if (opts->cpuset_clone_children) - set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags); -} - -static int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask) -{ - LIST_HEAD(tmp_links); - struct cgroup *root_cgrp = &root->cgrp; - struct css_set *cset; - int i, ret; - - lockdep_assert_held(&cgroup_mutex); - - ret = cgroup_idr_alloc(&root->cgroup_idr, root_cgrp, 1, 2, GFP_KERNEL); - if (ret < 0) - goto out; - root_cgrp->id = ret; - root_cgrp->ancestor_ids[0] = ret; - - ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release, 0, - GFP_KERNEL); - if (ret) - goto out; - - /* - * We're accessing css_set_count without locking css_set_lock here, - * but that's OK - it can only be increased by someone holding - * cgroup_lock, and that's us. Later rebinding may disable - * controllers on the default hierarchy and thus create new csets, - * which can't be more than the existing ones. Allocate 2x. - */ - ret = allocate_cgrp_cset_links(2 * css_set_count, &tmp_links); - if (ret) - goto cancel_ref; - - ret = cgroup_init_root_id(root); - if (ret) - goto cancel_ref; - - root->kf_root = kernfs_create_root(&cgroup_kf_syscall_ops, - KERNFS_ROOT_CREATE_DEACTIVATED, - root_cgrp); - if (IS_ERR(root->kf_root)) { - ret = PTR_ERR(root->kf_root); - goto exit_root_id; - } - root_cgrp->kn = root->kf_root->kn; - - ret = css_populate_dir(&root_cgrp->self); - if (ret) - goto destroy_root; - - ret = rebind_subsystems(root, ss_mask); - if (ret) - goto destroy_root; - - trace_cgroup_setup_root(root); - - /* - * There must be no failure case after here, since rebinding takes - * care of subsystems' refcounts, which are explicitly dropped in - * the failure exit path. - */ - list_add(&root->root_list, &cgroup_roots); - cgroup_root_count++; - - /* - * Link the root cgroup in this hierarchy into all the css_set - * objects. - */ - spin_lock_irq(&css_set_lock); - hash_for_each(css_set_table, i, cset, hlist) { - link_css_set(&tmp_links, cset, root_cgrp); - if (css_set_populated(cset)) - cgroup_update_populated(root_cgrp, true); - } - spin_unlock_irq(&css_set_lock); - - BUG_ON(!list_empty(&root_cgrp->self.children)); - BUG_ON(atomic_read(&root->nr_cgrps) != 1); - - kernfs_activate(root_cgrp->kn); - ret = 0; - goto out; - -destroy_root: - kernfs_destroy_root(root->kf_root); - root->kf_root = NULL; -exit_root_id: - cgroup_exit_root_id(root); -cancel_ref: - percpu_ref_exit(&root_cgrp->self.refcnt); -out: - free_cgrp_cset_links(&tmp_links); - return ret; -} - -static struct dentry *cgroup_mount(struct file_system_type *fs_type, - int flags, const char *unused_dev_name, - void *data) -{ - bool is_v2 = fs_type == &cgroup2_fs_type; - struct super_block *pinned_sb = NULL; - struct cgroup_namespace *ns = current->nsproxy->cgroup_ns; - struct cgroup_subsys *ss; - struct cgroup_root *root; - struct cgroup_sb_opts opts; - struct dentry *dentry; - int ret; - int i; - bool new_sb; - - get_cgroup_ns(ns); - - /* Check if the caller has permission to mount. */ - if (!ns_capable(ns->user_ns, CAP_SYS_ADMIN)) { - put_cgroup_ns(ns); - return ERR_PTR(-EPERM); - } - - /* - * The first time anyone tries to mount a cgroup, enable the list - * linking each css_set to its tasks and fix up all existing tasks. - */ - if (!use_task_css_set_links) - cgroup_enable_task_cg_lists(); - - if (is_v2) { - if (data) { - pr_err("cgroup2: unknown option \"%s\"\n", (char *)data); - put_cgroup_ns(ns); - return ERR_PTR(-EINVAL); - } - cgrp_dfl_visible = true; - root = &cgrp_dfl_root; - cgroup_get(&root->cgrp); - goto out_mount; - } - - cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); - - /* First find the desired set of subsystems */ - ret = parse_cgroupfs_options(data, &opts); - if (ret) - goto out_unlock; - - /* - * Destruction of cgroup root is asynchronous, so subsystems may - * still be dying after the previous unmount. Let's drain the - * dying subsystems. We just need to ensure that the ones - * unmounted previously finish dying and don't care about new ones - * starting. Testing ref liveliness is good enough. - */ - for_each_subsys(ss, i) { - if (!(opts.subsys_mask & (1 << i)) || - ss->root == &cgrp_dfl_root) - continue; - - if (!percpu_ref_tryget_live(&ss->root->cgrp.self.refcnt)) { - mutex_unlock(&cgroup_mutex); - msleep(10); - ret = restart_syscall(); - goto out_free; - } - cgroup_put(&ss->root->cgrp); - } - - for_each_root(root) { - bool name_match = false; - - if (root == &cgrp_dfl_root) - continue; - - /* - * If we asked for a name then it must match. Also, if - * name matches but sybsys_mask doesn't, we should fail. - * Remember whether name matched. - */ - if (opts.name) { - if (strcmp(opts.name, root->name)) - continue; - name_match = true; - } - - /* - * If we asked for subsystems (or explicitly for no - * subsystems) then they must match. - */ - if ((opts.subsys_mask || opts.none) && - (opts.subsys_mask != root->subsys_mask)) { - if (!name_match) - continue; - ret = -EBUSY; - goto out_unlock; - } - - if (root->flags ^ opts.flags) - pr_warn("new mount options do not match the existing superblock, will be ignored\n"); - - /* - * We want to reuse @root whose lifetime is governed by its - * ->cgrp. Let's check whether @root is alive and keep it - * that way. As cgroup_kill_sb() can happen anytime, we - * want to block it by pinning the sb so that @root doesn't - * get killed before mount is complete. - * - * With the sb pinned, tryget_live can reliably indicate - * whether @root can be reused. If it's being killed, - * drain it. We can use wait_queue for the wait but this - * path is super cold. Let's just sleep a bit and retry. - */ - pinned_sb = kernfs_pin_sb(root->kf_root, NULL); - if (IS_ERR(pinned_sb) || - !percpu_ref_tryget_live(&root->cgrp.self.refcnt)) { - mutex_unlock(&cgroup_mutex); - if (!IS_ERR_OR_NULL(pinned_sb)) - deactivate_super(pinned_sb); - msleep(10); - ret = restart_syscall(); - goto out_free; - } - - ret = 0; - goto out_unlock; - } - - /* - * No such thing, create a new one. name= matching without subsys - * specification is allowed for already existing hierarchies but we - * can't create new one without subsys specification. - */ - if (!opts.subsys_mask && !opts.none) { - ret = -EINVAL; - goto out_unlock; - } - - /* Hierarchies may only be created in the initial cgroup namespace. */ - if (ns != &init_cgroup_ns) { - ret = -EPERM; - goto out_unlock; - } - - root = kzalloc(sizeof(*root), GFP_KERNEL); - if (!root) { - ret = -ENOMEM; - goto out_unlock; - } - - init_cgroup_root(root, &opts); - - ret = cgroup_setup_root(root, opts.subsys_mask); - if (ret) - cgroup_free_root(root); - -out_unlock: - mutex_unlock(&cgroup_mutex); -out_free: - kfree(opts.release_agent); - kfree(opts.name); - - if (ret) { - put_cgroup_ns(ns); - return ERR_PTR(ret); - } -out_mount: - dentry = kernfs_mount(fs_type, flags, root->kf_root, - is_v2 ? CGROUP2_SUPER_MAGIC : CGROUP_SUPER_MAGIC, - &new_sb); - - /* - * In non-init cgroup namespace, instead of root cgroup's - * dentry, we return the dentry corresponding to the - * cgroupns->root_cgrp. - */ - if (!IS_ERR(dentry) && ns != &init_cgroup_ns) { - struct dentry *nsdentry; - struct cgroup *cgrp; - - mutex_lock(&cgroup_mutex); - spin_lock_irq(&css_set_lock); - - cgrp = cset_cgroup_from_root(ns->root_cset, root); - - spin_unlock_irq(&css_set_lock); - mutex_unlock(&cgroup_mutex); - - nsdentry = kernfs_node_dentry(cgrp->kn, dentry->d_sb); - dput(dentry); - dentry = nsdentry; - } - - if (IS_ERR(dentry) || !new_sb) - cgroup_put(&root->cgrp); - - /* - * If @pinned_sb, we're reusing an existing root and holding an - * extra ref on its sb. Mount is complete. Put the extra ref. - */ - if (pinned_sb) { - WARN_ON(new_sb); - deactivate_super(pinned_sb); - } - - put_cgroup_ns(ns); - return dentry; -} - -static void cgroup_kill_sb(struct super_block *sb) -{ - struct kernfs_root *kf_root = kernfs_root_from_sb(sb); - struct cgroup_root *root = cgroup_root_from_kf(kf_root); - - /* - * If @root doesn't have any mounts or children, start killing it. - * This prevents new mounts by disabling percpu_ref_tryget_live(). - * cgroup_mount() may wait for @root's release. - * - * And don't kill the default root. - */ - if (!list_empty(&root->cgrp.self.children) || - root == &cgrp_dfl_root) - cgroup_put(&root->cgrp); - else - percpu_ref_kill(&root->cgrp.self.refcnt); - - kernfs_kill_sb(sb); -} - -static struct file_system_type cgroup_fs_type = { - .name = "cgroup", - .mount = cgroup_mount, - .kill_sb = cgroup_kill_sb, - .fs_flags = FS_USERNS_MOUNT, -}; - -static struct file_system_type cgroup2_fs_type = { - .name = "cgroup2", - .mount = cgroup_mount, - .kill_sb = cgroup_kill_sb, - .fs_flags = FS_USERNS_MOUNT, -}; - -static int cgroup_path_ns_locked(struct cgroup *cgrp, char *buf, size_t buflen, - struct cgroup_namespace *ns) -{ - struct cgroup *root = cset_cgroup_from_root(ns->root_cset, cgrp->root); - - return kernfs_path_from_node(cgrp->kn, root->kn, buf, buflen); -} - -int cgroup_path_ns(struct cgroup *cgrp, char *buf, size_t buflen, - struct cgroup_namespace *ns) -{ - int ret; - - mutex_lock(&cgroup_mutex); - spin_lock_irq(&css_set_lock); - - ret = cgroup_path_ns_locked(cgrp, buf, buflen, ns); - - spin_unlock_irq(&css_set_lock); - mutex_unlock(&cgroup_mutex); - - return ret; -} -EXPORT_SYMBOL_GPL(cgroup_path_ns); - -/** - * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy - * @task: target task - * @buf: the buffer to write the path into - * @buflen: the length of the buffer - * - * Determine @task's cgroup on the first (the one with the lowest non-zero - * hierarchy_id) cgroup hierarchy and copy its path into @buf. This - * function grabs cgroup_mutex and shouldn't be used inside locks used by - * cgroup controller callbacks. - * - * Return value is the same as kernfs_path(). - */ -int task_cgroup_path(struct task_struct *task, char *buf, size_t buflen) -{ - struct cgroup_root *root; - struct cgroup *cgrp; - int hierarchy_id = 1; - int ret; - - mutex_lock(&cgroup_mutex); - spin_lock_irq(&css_set_lock); - - root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id); - - if (root) { - cgrp = task_cgroup_from_root(task, root); - ret = cgroup_path_ns_locked(cgrp, buf, buflen, &init_cgroup_ns); - } else { - /* if no hierarchy exists, everyone is in "/" */ - ret = strlcpy(buf, "/", buflen); - } - - spin_unlock_irq(&css_set_lock); - mutex_unlock(&cgroup_mutex); - return ret; -} -EXPORT_SYMBOL_GPL(task_cgroup_path); - -/* used to track tasks and other necessary states during migration */ -struct cgroup_taskset { - /* the src and dst cset list running through cset->mg_node */ - struct list_head src_csets; - struct list_head dst_csets; - - /* the subsys currently being processed */ - int ssid; - - /* - * Fields for cgroup_taskset_*() iteration. - * - * Before migration is committed, the target migration tasks are on - * ->mg_tasks of the csets on ->src_csets. After, on ->mg_tasks of - * the csets on ->dst_csets. ->csets point to either ->src_csets - * or ->dst_csets depending on whether migration is committed. - * - * ->cur_csets and ->cur_task point to the current task position - * during iteration. - */ - struct list_head *csets; - struct css_set *cur_cset; - struct task_struct *cur_task; -}; - -#define CGROUP_TASKSET_INIT(tset) (struct cgroup_taskset){ \ - .src_csets = LIST_HEAD_INIT(tset.src_csets), \ - .dst_csets = LIST_HEAD_INIT(tset.dst_csets), \ - .csets = &tset.src_csets, \ -} - -/** - * cgroup_taskset_add - try to add a migration target task to a taskset - * @task: target task - * @tset: target taskset - * - * Add @task, which is a migration target, to @tset. This function becomes - * noop if @task doesn't need to be migrated. @task's css_set should have - * been added as a migration source and @task->cg_list will be moved from - * the css_set's tasks list to mg_tasks one. - */ -static void cgroup_taskset_add(struct task_struct *task, - struct cgroup_taskset *tset) -{ - struct css_set *cset; - - lockdep_assert_held(&css_set_lock); - - /* @task either already exited or can't exit until the end */ - if (task->flags & PF_EXITING) - return; - - /* leave @task alone if post_fork() hasn't linked it yet */ - if (list_empty(&task->cg_list)) - return; - - cset = task_css_set(task); - if (!cset->mg_src_cgrp) - return; - - list_move_tail(&task->cg_list, &cset->mg_tasks); - if (list_empty(&cset->mg_node)) - list_add_tail(&cset->mg_node, &tset->src_csets); - if (list_empty(&cset->mg_dst_cset->mg_node)) - list_move_tail(&cset->mg_dst_cset->mg_node, - &tset->dst_csets); -} - -/** - * cgroup_taskset_first - reset taskset and return the first task - * @tset: taskset of interest - * @dst_cssp: output variable for the destination css - * - * @tset iteration is initialized and the first task is returned. - */ -struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset, - struct cgroup_subsys_state **dst_cssp) -{ - tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node); - tset->cur_task = NULL; - - return cgroup_taskset_next(tset, dst_cssp); -} - -/** - * cgroup_taskset_next - iterate to the next task in taskset - * @tset: taskset of interest - * @dst_cssp: output variable for the destination css - * - * Return the next task in @tset. Iteration must have been initialized - * with cgroup_taskset_first(). - */ -struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset, - struct cgroup_subsys_state **dst_cssp) -{ - struct css_set *cset = tset->cur_cset; - struct task_struct *task = tset->cur_task; - - while (&cset->mg_node != tset->csets) { - if (!task) - task = list_first_entry(&cset->mg_tasks, - struct task_struct, cg_list); - else - task = list_next_entry(task, cg_list); - - if (&task->cg_list != &cset->mg_tasks) { - tset->cur_cset = cset; - tset->cur_task = task; - - /* - * This function may be called both before and - * after cgroup_taskset_migrate(). The two cases - * can be distinguished by looking at whether @cset - * has its ->mg_dst_cset set. - */ - if (cset->mg_dst_cset) - *dst_cssp = cset->mg_dst_cset->subsys[tset->ssid]; - else - *dst_cssp = cset->subsys[tset->ssid]; - - return task; - } - - cset = list_next_entry(cset, mg_node); - task = NULL; - } - - return NULL; -} - -/** - * cgroup_taskset_migrate - migrate a taskset - * @tset: taget taskset - * @root: cgroup root the migration is taking place on - * - * Migrate tasks in @tset as setup by migration preparation functions. - * This function fails iff one of the ->can_attach callbacks fails and - * guarantees that either all or none of the tasks in @tset are migrated. - * @tset is consumed regardless of success. - */ -static int cgroup_taskset_migrate(struct cgroup_taskset *tset, - struct cgroup_root *root) -{ - struct cgroup_subsys *ss; - struct task_struct *task, *tmp_task; - struct css_set *cset, *tmp_cset; - int ssid, failed_ssid, ret; - - /* methods shouldn't be called if no task is actually migrating */ - if (list_empty(&tset->src_csets)) - return 0; - - /* check that we can legitimately attach to the cgroup */ - do_each_subsys_mask(ss, ssid, root->subsys_mask) { - if (ss->can_attach) { - tset->ssid = ssid; - ret = ss->can_attach(tset); - if (ret) { - failed_ssid = ssid; - goto out_cancel_attach; - } - } - } while_each_subsys_mask(); - - /* - * Now that we're guaranteed success, proceed to move all tasks to - * the new cgroup. There are no failure cases after here, so this - * is the commit point. - */ - spin_lock_irq(&css_set_lock); - list_for_each_entry(cset, &tset->src_csets, mg_node) { - list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) { - struct css_set *from_cset = task_css_set(task); - struct css_set *to_cset = cset->mg_dst_cset; - - get_css_set(to_cset); - css_set_move_task(task, from_cset, to_cset, true); - put_css_set_locked(from_cset); - } - } - spin_unlock_irq(&css_set_lock); - - /* - * Migration is committed, all target tasks are now on dst_csets. - * Nothing is sensitive to fork() after this point. Notify - * controllers that migration is complete. - */ - tset->csets = &tset->dst_csets; - - do_each_subsys_mask(ss, ssid, root->subsys_mask) { - if (ss->attach) { - tset->ssid = ssid; - ss->attach(tset); - } - } while_each_subsys_mask(); - - ret = 0; - goto out_release_tset; - -out_cancel_attach: - do_each_subsys_mask(ss, ssid, root->subsys_mask) { - if (ssid == failed_ssid) - break; - if (ss->cancel_attach) { - tset->ssid = ssid; - ss->cancel_attach(tset); - } - } while_each_subsys_mask(); -out_release_tset: - spin_lock_irq(&css_set_lock); - list_splice_init(&tset->dst_csets, &tset->src_csets); - list_for_each_entry_safe(cset, tmp_cset, &tset->src_csets, mg_node) { - list_splice_tail_init(&cset->mg_tasks, &cset->tasks); - list_del_init(&cset->mg_node); - } - spin_unlock_irq(&css_set_lock); - return ret; -} - -/** - * cgroup_may_migrate_to - verify whether a cgroup can be migration destination - * @dst_cgrp: destination cgroup to test - * - * On the default hierarchy, except for the root, subtree_control must be - * zero for migration destination cgroups with tasks so that child cgroups - * don't compete against tasks. - */ -static bool cgroup_may_migrate_to(struct cgroup *dst_cgrp) -{ - return !cgroup_on_dfl(dst_cgrp) || !cgroup_parent(dst_cgrp) || - !dst_cgrp->subtree_control; -} - -/** - * cgroup_migrate_finish - cleanup after attach - * @preloaded_csets: list of preloaded css_sets - * - * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst(). See - * those functions for details. - */ -static void cgroup_migrate_finish(struct list_head *preloaded_csets) -{ - struct css_set *cset, *tmp_cset; - - lockdep_assert_held(&cgroup_mutex); - - spin_lock_irq(&css_set_lock); - list_for_each_entry_safe(cset, tmp_cset, preloaded_csets, mg_preload_node) { - cset->mg_src_cgrp = NULL; - cset->mg_dst_cgrp = NULL; - cset->mg_dst_cset = NULL; - list_del_init(&cset->mg_preload_node); - put_css_set_locked(cset); - } - spin_unlock_irq(&css_set_lock); -} - -/** - * cgroup_migrate_add_src - add a migration source css_set - * @src_cset: the source css_set to add - * @dst_cgrp: the destination cgroup - * @preloaded_csets: list of preloaded css_sets - * - * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp. Pin - * @src_cset and add it to @preloaded_csets, which should later be cleaned - * up by cgroup_migrate_finish(). - * - * This function may be called without holding cgroup_threadgroup_rwsem - * even if the target is a process. Threads may be created and destroyed - * but as long as cgroup_mutex is not dropped, no new css_set can be put - * into play and the preloaded css_sets are guaranteed to cover all - * migrations. - */ -static void cgroup_migrate_add_src(struct css_set *src_cset, - struct cgroup *dst_cgrp, - struct list_head *preloaded_csets) -{ - struct cgroup *src_cgrp; - - lockdep_assert_held(&cgroup_mutex); - lockdep_assert_held(&css_set_lock); - - /* - * If ->dead, @src_set is associated with one or more dead cgroups - * and doesn't contain any migratable tasks. Ignore it early so - * that the rest of migration path doesn't get confused by it. - */ - if (src_cset->dead) - return; - - src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root); - - if (!list_empty(&src_cset->mg_preload_node)) - return; - - WARN_ON(src_cset->mg_src_cgrp); - WARN_ON(src_cset->mg_dst_cgrp); - WARN_ON(!list_empty(&src_cset->mg_tasks)); - WARN_ON(!list_empty(&src_cset->mg_node)); - - src_cset->mg_src_cgrp = src_cgrp; - src_cset->mg_dst_cgrp = dst_cgrp; - get_css_set(src_cset); - list_add(&src_cset->mg_preload_node, preloaded_csets); -} - -/** - * cgroup_migrate_prepare_dst - prepare destination css_sets for migration - * @preloaded_csets: list of preloaded source css_sets - * - * Tasks are about to be moved and all the source css_sets have been - * preloaded to @preloaded_csets. This function looks up and pins all - * destination css_sets, links each to its source, and append them to - * @preloaded_csets. - * - * This function must be called after cgroup_migrate_add_src() has been - * called on each migration source css_set. After migration is performed - * using cgroup_migrate(), cgroup_migrate_finish() must be called on - * @preloaded_csets. - */ -static int cgroup_migrate_prepare_dst(struct list_head *preloaded_csets) -{ - LIST_HEAD(csets); - struct css_set *src_cset, *tmp_cset; - - lockdep_assert_held(&cgroup_mutex); - - /* look up the dst cset for each src cset and link it to src */ - list_for_each_entry_safe(src_cset, tmp_cset, preloaded_csets, mg_preload_node) { - struct css_set *dst_cset; - - dst_cset = find_css_set(src_cset, src_cset->mg_dst_cgrp); - if (!dst_cset) - goto err; - - WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset); - - /* - * If src cset equals dst, it's noop. Drop the src. - * cgroup_migrate() will skip the cset too. Note that we - * can't handle src == dst as some nodes are used by both. - */ - if (src_cset == dst_cset) { - src_cset->mg_src_cgrp = NULL; - src_cset->mg_dst_cgrp = NULL; - list_del_init(&src_cset->mg_preload_node); - put_css_set(src_cset); - put_css_set(dst_cset); - continue; - } - - src_cset->mg_dst_cset = dst_cset; - - if (list_empty(&dst_cset->mg_preload_node)) - list_add(&dst_cset->mg_preload_node, &csets); - else - put_css_set(dst_cset); - } - - list_splice_tail(&csets, preloaded_csets); - return 0; -err: - cgroup_migrate_finish(&csets); - return -ENOMEM; -} - -/** - * cgroup_migrate - migrate a process or task to a cgroup - * @leader: the leader of the process or the task to migrate - * @threadgroup: whether @leader points to the whole process or a single task - * @root: cgroup root migration is taking place on - * - * Migrate a process or task denoted by @leader. If migrating a process, - * the caller must be holding cgroup_threadgroup_rwsem. The caller is also - * responsible for invoking cgroup_migrate_add_src() and - * cgroup_migrate_prepare_dst() on the targets before invoking this - * function and following up with cgroup_migrate_finish(). - * - * As long as a controller's ->can_attach() doesn't fail, this function is - * guaranteed to succeed. This means that, excluding ->can_attach() - * failure, when migrating multiple targets, the success or failure can be - * decided for all targets by invoking group_migrate_prepare_dst() before - * actually starting migrating. - */ -static int cgroup_migrate(struct task_struct *leader, bool threadgroup, - struct cgroup_root *root) -{ - struct cgroup_taskset tset = CGROUP_TASKSET_INIT(tset); - struct task_struct *task; - - /* - * Prevent freeing of tasks while we take a snapshot. Tasks that are - * already PF_EXITING could be freed from underneath us unless we - * take an rcu_read_lock. - */ - spin_lock_irq(&css_set_lock); - rcu_read_lock(); - task = leader; - do { - cgroup_taskset_add(task, &tset); - if (!threadgroup) - break; - } while_each_thread(leader, task); - rcu_read_unlock(); - spin_unlock_irq(&css_set_lock); - - return cgroup_taskset_migrate(&tset, root); -} - -/** - * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup - * @dst_cgrp: the cgroup to attach to - * @leader: the task or the leader of the threadgroup to be attached - * @threadgroup: attach the whole threadgroup? - * - * Call holding cgroup_mutex and cgroup_threadgroup_rwsem. - */ -static int cgroup_attach_task(struct cgroup *dst_cgrp, - struct task_struct *leader, bool threadgroup) -{ - LIST_HEAD(preloaded_csets); - struct task_struct *task; - int ret; - - if (!cgroup_may_migrate_to(dst_cgrp)) - return -EBUSY; - - /* look up all src csets */ - spin_lock_irq(&css_set_lock); - rcu_read_lock(); - task = leader; - do { - cgroup_migrate_add_src(task_css_set(task), dst_cgrp, - &preloaded_csets); - if (!threadgroup) - break; - } while_each_thread(leader, task); - rcu_read_unlock(); - spin_unlock_irq(&css_set_lock); - - /* prepare dst csets and commit */ - ret = cgroup_migrate_prepare_dst(&preloaded_csets); - if (!ret) - ret = cgroup_migrate(leader, threadgroup, dst_cgrp->root); - - cgroup_migrate_finish(&preloaded_csets); - - if (!ret) - trace_cgroup_attach_task(dst_cgrp, leader, threadgroup); - - return ret; -} - -static int cgroup_procs_write_permission(struct task_struct *task, - struct cgroup *dst_cgrp, - struct kernfs_open_file *of) -{ - const struct cred *cred = current_cred(); - const struct cred *tcred = get_task_cred(task); - int ret = 0; - - /* - * even if we're attaching all tasks in the thread group, we only - * need to check permissions on one of them. - */ - if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) && - !uid_eq(cred->euid, tcred->uid) && - !uid_eq(cred->euid, tcred->suid)) - ret = -EACCES; - - if (!ret && cgroup_on_dfl(dst_cgrp)) { - struct super_block *sb = of->file->f_path.dentry->d_sb; - struct cgroup *cgrp; - struct inode *inode; - - spin_lock_irq(&css_set_lock); - cgrp = task_cgroup_from_root(task, &cgrp_dfl_root); - spin_unlock_irq(&css_set_lock); - - while (!cgroup_is_descendant(dst_cgrp, cgrp)) - cgrp = cgroup_parent(cgrp); - - ret = -ENOMEM; - inode = kernfs_get_inode(sb, cgrp->procs_file.kn); - if (inode) { - ret = inode_permission(inode, MAY_WRITE); - iput(inode); - } - } - - put_cred(tcred); - return ret; -} - -/* - * Find the task_struct of the task to attach by vpid and pass it along to the - * function to attach either it or all tasks in its threadgroup. Will lock - * cgroup_mutex and threadgroup. - */ -static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf, - size_t nbytes, loff_t off, bool threadgroup) -{ - struct task_struct *tsk; - struct cgroup_subsys *ss; - struct cgroup *cgrp; - pid_t pid; - int ssid, ret; - - if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0) - return -EINVAL; - - cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; - - percpu_down_write(&cgroup_threadgroup_rwsem); - rcu_read_lock(); - if (pid) { - tsk = find_task_by_vpid(pid); - if (!tsk) { - ret = -ESRCH; - goto out_unlock_rcu; - } - } else { - tsk = current; - } - - if (threadgroup) - tsk = tsk->group_leader; - - /* - * Workqueue threads may acquire PF_NO_SETAFFINITY and become - * trapped in a cpuset, or RT worker may be born in a cgroup - * with no rt_runtime allocated. Just say no. - */ - if (tsk == kthreadd_task || (tsk->flags & PF_NO_SETAFFINITY)) { - ret = -EINVAL; - goto out_unlock_rcu; - } - - get_task_struct(tsk); - rcu_read_unlock(); - - ret = cgroup_procs_write_permission(tsk, cgrp, of); - if (!ret) - ret = cgroup_attach_task(cgrp, tsk, threadgroup); - - put_task_struct(tsk); - goto out_unlock_threadgroup; - -out_unlock_rcu: - rcu_read_unlock(); -out_unlock_threadgroup: - percpu_up_write(&cgroup_threadgroup_rwsem); - for_each_subsys(ss, ssid) - if (ss->post_attach) - ss->post_attach(); - cgroup_kn_unlock(of->kn); - return ret ?: nbytes; -} - -/** - * cgroup_attach_task_all - attach task 'tsk' to all cgroups of task 'from' - * @from: attach to all cgroups of a given task - * @tsk: the task to be attached - */ -int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk) -{ - struct cgroup_root *root; - int retval = 0; - - mutex_lock(&cgroup_mutex); - percpu_down_write(&cgroup_threadgroup_rwsem); - for_each_root(root) { - struct cgroup *from_cgrp; - - if (root == &cgrp_dfl_root) - continue; - - spin_lock_irq(&css_set_lock); - from_cgrp = task_cgroup_from_root(from, root); - spin_unlock_irq(&css_set_lock); - - retval = cgroup_attach_task(from_cgrp, tsk, false); - if (retval) - break; - } - percpu_up_write(&cgroup_threadgroup_rwsem); - mutex_unlock(&cgroup_mutex); - - return retval; -} -EXPORT_SYMBOL_GPL(cgroup_attach_task_all); - -static ssize_t cgroup_tasks_write(struct kernfs_open_file *of, - char *buf, size_t nbytes, loff_t off) -{ - return __cgroup_procs_write(of, buf, nbytes, off, false); -} - -static ssize_t cgroup_procs_write(struct kernfs_open_file *of, - char *buf, size_t nbytes, loff_t off) -{ - return __cgroup_procs_write(of, buf, nbytes, off, true); -} - -static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of, - char *buf, size_t nbytes, loff_t off) -{ - struct cgroup *cgrp; - - BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX); - - cgrp = cgroup_kn_lock_live(of->kn, false); - if (!cgrp) - return -ENODEV; - spin_lock(&release_agent_path_lock); - strlcpy(cgrp->root->release_agent_path, strstrip(buf), - sizeof(cgrp->root->release_agent_path)); - spin_unlock(&release_agent_path_lock); - cgroup_kn_unlock(of->kn); - return nbytes; -} - -static int cgroup_release_agent_show(struct seq_file *seq, void *v) -{ - struct cgroup *cgrp = seq_css(seq)->cgroup; - - spin_lock(&release_agent_path_lock); - seq_puts(seq, cgrp->root->release_agent_path); - spin_unlock(&release_agent_path_lock); - seq_putc(seq, '\n'); - return 0; -} - -static int cgroup_sane_behavior_show(struct seq_file *seq, void *v) -{ - seq_puts(seq, "0\n"); - return 0; -} - -static void cgroup_print_ss_mask(struct seq_file *seq, u16 ss_mask) -{ - struct cgroup_subsys *ss; - bool printed = false; - int ssid; - - do_each_subsys_mask(ss, ssid, ss_mask) { - if (printed) - seq_putc(seq, ' '); - seq_printf(seq, "%s", ss->name); - printed = true; - } while_each_subsys_mask(); - if (printed) - seq_putc(seq, '\n'); -} - -/* show controllers which are enabled from the parent */ -static int cgroup_controllers_show(struct seq_file *seq, void *v) -{ - struct cgroup *cgrp = seq_css(seq)->cgroup; - - cgroup_print_ss_mask(seq, cgroup_control(cgrp)); - return 0; -} - -/* show controllers which are enabled for a given cgroup's children */ -static int cgroup_subtree_control_show(struct seq_file *seq, void *v) -{ - struct cgroup *cgrp = seq_css(seq)->cgroup; - - cgroup_print_ss_mask(seq, cgrp->subtree_control); - return 0; -} - -/** - * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy - * @cgrp: root of the subtree to update csses for - * - * @cgrp's control masks have changed and its subtree's css associations - * need to be updated accordingly. This function looks up all css_sets - * which are attached to the subtree, creates the matching updated css_sets - * and migrates the tasks to the new ones. - */ -static int cgroup_update_dfl_csses(struct cgroup *cgrp) -{ - LIST_HEAD(preloaded_csets); - struct cgroup_taskset tset = CGROUP_TASKSET_INIT(tset); - struct cgroup_subsys_state *d_css; - struct cgroup *dsct; - struct css_set *src_cset; - int ret; - - lockdep_assert_held(&cgroup_mutex); - - percpu_down_write(&cgroup_threadgroup_rwsem); - - /* look up all csses currently attached to @cgrp's subtree */ - spin_lock_irq(&css_set_lock); - cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { - struct cgrp_cset_link *link; - - list_for_each_entry(link, &dsct->cset_links, cset_link) - cgroup_migrate_add_src(link->cset, dsct, - &preloaded_csets); - } - spin_unlock_irq(&css_set_lock); - - /* NULL dst indicates self on default hierarchy */ - ret = cgroup_migrate_prepare_dst(&preloaded_csets); - if (ret) - goto out_finish; - - spin_lock_irq(&css_set_lock); - list_for_each_entry(src_cset, &preloaded_csets, mg_preload_node) { - struct task_struct *task, *ntask; - - /* src_csets precede dst_csets, break on the first dst_cset */ - if (!src_cset->mg_src_cgrp) - break; - - /* all tasks in src_csets need to be migrated */ - list_for_each_entry_safe(task, ntask, &src_cset->tasks, cg_list) - cgroup_taskset_add(task, &tset); - } - spin_unlock_irq(&css_set_lock); - - ret = cgroup_taskset_migrate(&tset, cgrp->root); -out_finish: - cgroup_migrate_finish(&preloaded_csets); - percpu_up_write(&cgroup_threadgroup_rwsem); - return ret; -} - -/** - * cgroup_lock_and_drain_offline - lock cgroup_mutex and drain offlined csses - * @cgrp: root of the target subtree - * - * Because css offlining is asynchronous, userland may try to re-enable a - * controller while the previous css is still around. This function grabs - * cgroup_mutex and drains the previous css instances of @cgrp's subtree. - */ -static void cgroup_lock_and_drain_offline(struct cgroup *cgrp) - __acquires(&cgroup_mutex) -{ - struct cgroup *dsct; - struct cgroup_subsys_state *d_css; - struct cgroup_subsys *ss; - int ssid; - -restart: - mutex_lock(&cgroup_mutex); - - cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { - for_each_subsys(ss, ssid) { - struct cgroup_subsys_state *css = cgroup_css(dsct, ss); - DEFINE_WAIT(wait); - - if (!css || !percpu_ref_is_dying(&css->refcnt)) - continue; - - cgroup_get(dsct); - prepare_to_wait(&dsct->offline_waitq, &wait, - TASK_UNINTERRUPTIBLE); - - mutex_unlock(&cgroup_mutex); - schedule(); - finish_wait(&dsct->offline_waitq, &wait); - - cgroup_put(dsct); - goto restart; - } - } -} - -/** - * cgroup_save_control - save control masks of a subtree - * @cgrp: root of the target subtree - * - * Save ->subtree_control and ->subtree_ss_mask to the respective old_ - * prefixed fields for @cgrp's subtree including @cgrp itself. - */ -static void cgroup_save_control(struct cgroup *cgrp) -{ - struct cgroup *dsct; - struct cgroup_subsys_state *d_css; - - cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { - dsct->old_subtree_control = dsct->subtree_control; - dsct->old_subtree_ss_mask = dsct->subtree_ss_mask; - } -} - -/** - * cgroup_propagate_control - refresh control masks of a subtree - * @cgrp: root of the target subtree - * - * For @cgrp and its subtree, ensure ->subtree_ss_mask matches - * ->subtree_control and propagate controller availability through the - * subtree so that descendants don't have unavailable controllers enabled. - */ -static void cgroup_propagate_control(struct cgroup *cgrp) -{ - struct cgroup *dsct; - struct cgroup_subsys_state *d_css; - - cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { - dsct->subtree_control &= cgroup_control(dsct); - dsct->subtree_ss_mask = - cgroup_calc_subtree_ss_mask(dsct->subtree_control, - cgroup_ss_mask(dsct)); - } -} - -/** - * cgroup_restore_control - restore control masks of a subtree - * @cgrp: root of the target subtree - * - * Restore ->subtree_control and ->subtree_ss_mask from the respective old_ - * prefixed fields for @cgrp's subtree including @cgrp itself. - */ -static void cgroup_restore_control(struct cgroup *cgrp) -{ - struct cgroup *dsct; - struct cgroup_subsys_state *d_css; - - cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { - dsct->subtree_control = dsct->old_subtree_control; - dsct->subtree_ss_mask = dsct->old_subtree_ss_mask; - } -} - -static bool css_visible(struct cgroup_subsys_state *css) -{ - struct cgroup_subsys *ss = css->ss; - struct cgroup *cgrp = css->cgroup; - - if (cgroup_control(cgrp) & (1 << ss->id)) - return true; - if (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) - return false; - return cgroup_on_dfl(cgrp) && ss->implicit_on_dfl; -} - -/** - * cgroup_apply_control_enable - enable or show csses according to control - * @cgrp: root of the target subtree - * - * Walk @cgrp's subtree and create new csses or make the existing ones - * visible. A css is created invisible if it's being implicitly enabled - * through dependency. An invisible css is made visible when the userland - * explicitly enables it. - * - * Returns 0 on success, -errno on failure. On failure, csses which have - * been processed already aren't cleaned up. The caller is responsible for - * cleaning up with cgroup_apply_control_disble(). - */ -static int cgroup_apply_control_enable(struct cgroup *cgrp) -{ - struct cgroup *dsct; - struct cgroup_subsys_state *d_css; - struct cgroup_subsys *ss; - int ssid, ret; - - cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { - for_each_subsys(ss, ssid) { - struct cgroup_subsys_state *css = cgroup_css(dsct, ss); - - WARN_ON_ONCE(css && percpu_ref_is_dying(&css->refcnt)); - - if (!(cgroup_ss_mask(dsct) & (1 << ss->id))) - continue; - - if (!css) { - css = css_create(dsct, ss); - if (IS_ERR(css)) - return PTR_ERR(css); - } - - if (css_visible(css)) { - ret = css_populate_dir(css); - if (ret) - return ret; - } - } - } - - return 0; -} - -/** - * cgroup_apply_control_disable - kill or hide csses according to control - * @cgrp: root of the target subtree - * - * Walk @cgrp's subtree and kill and hide csses so that they match - * cgroup_ss_mask() and cgroup_visible_mask(). - * - * A css is hidden when the userland requests it to be disabled while other - * subsystems are still depending on it. The css must not actively control - * resources and be in the vanilla state if it's made visible again later. - * Controllers which may be depended upon should provide ->css_reset() for - * this purpose. - */ -static void cgroup_apply_control_disable(struct cgroup *cgrp) -{ - struct cgroup *dsct; - struct cgroup_subsys_state *d_css; - struct cgroup_subsys *ss; - int ssid; - - cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { - for_each_subsys(ss, ssid) { - struct cgroup_subsys_state *css = cgroup_css(dsct, ss); - - WARN_ON_ONCE(css && percpu_ref_is_dying(&css->refcnt)); - - if (!css) - continue; - - if (css->parent && - !(cgroup_ss_mask(dsct) & (1 << ss->id))) { - kill_css(css); - } else if (!css_visible(css)) { - css_clear_dir(css); - if (ss->css_reset) - ss->css_reset(css); - } - } - } -} - -/** - * cgroup_apply_control - apply control mask updates to the subtree - * @cgrp: root of the target subtree - * - * subsystems can be enabled and disabled in a subtree using the following - * steps. - * - * 1. Call cgroup_save_control() to stash the current state. - * 2. Update ->subtree_control masks in the subtree as desired. - * 3. Call cgroup_apply_control() to apply the changes. - * 4. Optionally perform other related operations. - * 5. Call cgroup_finalize_control() to finish up. - * - * This function implements step 3 and propagates the mask changes - * throughout @cgrp's subtree, updates csses accordingly and perform - * process migrations. - */ -static int cgroup_apply_control(struct cgroup *cgrp) -{ - int ret; - - cgroup_propagate_control(cgrp); - - ret = cgroup_apply_control_enable(cgrp); - if (ret) - return ret; - - /* - * At this point, cgroup_e_css() results reflect the new csses - * making the following cgroup_update_dfl_csses() properly update - * css associations of all tasks in the subtree. - */ - ret = cgroup_update_dfl_csses(cgrp); - if (ret) - return ret; - - return 0; -} - -/** - * cgroup_finalize_control - finalize control mask update - * @cgrp: root of the target subtree - * @ret: the result of the update - * - * Finalize control mask update. See cgroup_apply_control() for more info. - */ -static void cgroup_finalize_control(struct cgroup *cgrp, int ret) -{ - if (ret) { - cgroup_restore_control(cgrp); - cgroup_propagate_control(cgrp); - } - - cgroup_apply_control_disable(cgrp); -} - -/* change the enabled child controllers for a cgroup in the default hierarchy */ -static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of, - char *buf, size_t nbytes, - loff_t off) -{ - u16 enable = 0, disable = 0; - struct cgroup *cgrp, *child; - struct cgroup_subsys *ss; - char *tok; - int ssid, ret; - - /* - * Parse input - space separated list of subsystem names prefixed - * with either + or -. - */ - buf = strstrip(buf); - while ((tok = strsep(&buf, " "))) { - if (tok[0] == '\0') - continue; - do_each_subsys_mask(ss, ssid, ~cgrp_dfl_inhibit_ss_mask) { - if (!cgroup_ssid_enabled(ssid) || - strcmp(tok + 1, ss->name)) - continue; - - if (*tok == '+') { - enable |= 1 << ssid; - disable &= ~(1 << ssid); - } else if (*tok == '-') { - disable |= 1 << ssid; - enable &= ~(1 << ssid); - } else { - return -EINVAL; - } - break; - } while_each_subsys_mask(); - if (ssid == CGROUP_SUBSYS_COUNT) - return -EINVAL; - } - - cgrp = cgroup_kn_lock_live(of->kn, true); - if (!cgrp) - return -ENODEV; - - for_each_subsys(ss, ssid) { - if (enable & (1 << ssid)) { - if (cgrp->subtree_control & (1 << ssid)) { - enable &= ~(1 << ssid); - continue; - } - - if (!(cgroup_control(cgrp) & (1 << ssid))) { - ret = -ENOENT; - goto out_unlock; - } - } else if (disable & (1 << ssid)) { - if (!(cgrp->subtree_control & (1 << ssid))) { - disable &= ~(1 << ssid); - continue; - } - - /* a child has it enabled? */ - cgroup_for_each_live_child(child, cgrp) { - if (child->subtree_control & (1 << ssid)) { - ret = -EBUSY; - goto out_unlock; - } - } - } - } - - if (!enable && !disable) { - ret = 0; - goto out_unlock; - } - - /* - * Except for the root, subtree_control must be zero for a cgroup - * with tasks so that child cgroups don't compete against tasks. - */ - if (enable && cgroup_parent(cgrp)) { - struct cgrp_cset_link *link; - - /* - * Because namespaces pin csets too, @cgrp->cset_links - * might not be empty even when @cgrp is empty. Walk and - * verify each cset. - */ - spin_lock_irq(&css_set_lock); - - ret = 0; - list_for_each_entry(link, &cgrp->cset_links, cset_link) { - if (css_set_populated(link->cset)) { - ret = -EBUSY; - break; - } - } - - spin_unlock_irq(&css_set_lock); - - if (ret) - goto out_unlock; - } - - /* save and update control masks and prepare csses */ - cgroup_save_control(cgrp); - - cgrp->subtree_control |= enable; - cgrp->subtree_control &= ~disable; - - ret = cgroup_apply_control(cgrp); - - cgroup_finalize_control(cgrp, ret); - - kernfs_activate(cgrp->kn); - ret = 0; -out_unlock: - cgroup_kn_unlock(of->kn); - return ret ?: nbytes; -} - -static int cgroup_events_show(struct seq_file *seq, void *v) -{ - seq_printf(seq, "populated %d\n", - cgroup_is_populated(seq_css(seq)->cgroup)); - return 0; -} - -static int cgroup_file_open(struct kernfs_open_file *of) -{ - struct cftype *cft = of->kn->priv; - - if (cft->open) - return cft->open(of); - return 0; -} - -static void cgroup_file_release(struct kernfs_open_file *of) -{ - struct cftype *cft = of->kn->priv; - - if (cft->release) - cft->release(of); -} - -static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf, - size_t nbytes, loff_t off) -{ - struct cgroup *cgrp = of->kn->parent->priv; - struct cftype *cft = of->kn->priv; - struct cgroup_subsys_state *css; - int ret; - - if (cft->write) - return cft->write(of, buf, nbytes, off); - - /* - * kernfs guarantees that a file isn't deleted with operations in - * flight, which means that the matching css is and stays alive and - * doesn't need to be pinned. The RCU locking is not necessary - * either. It's just for the convenience of using cgroup_css(). - */ - rcu_read_lock(); - css = cgroup_css(cgrp, cft->ss); - rcu_read_unlock(); - - if (cft->write_u64) { - unsigned long long v; - ret = kstrtoull(buf, 0, &v); - if (!ret) - ret = cft->write_u64(css, cft, v); - } else if (cft->write_s64) { - long long v; - ret = kstrtoll(buf, 0, &v); - if (!ret) - ret = cft->write_s64(css, cft, v); - } else { - ret = -EINVAL; - } - - return ret ?: nbytes; -} - -static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos) -{ - return seq_cft(seq)->seq_start(seq, ppos); -} - -static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos) -{ - return seq_cft(seq)->seq_next(seq, v, ppos); -} - -static void cgroup_seqfile_stop(struct seq_file *seq, void *v) -{ - if (seq_cft(seq)->seq_stop) - seq_cft(seq)->seq_stop(seq, v); -} - -static int cgroup_seqfile_show(struct seq_file *m, void *arg) -{ - struct cftype *cft = seq_cft(m); - struct cgroup_subsys_state *css = seq_css(m); - - if (cft->seq_show) - return cft->seq_show(m, arg); - - if (cft->read_u64) - seq_printf(m, "%llu\n", cft->read_u64(css, cft)); - else if (cft->read_s64) - seq_printf(m, "%lld\n", cft->read_s64(css, cft)); - else - return -EINVAL; - return 0; -} - -static struct kernfs_ops cgroup_kf_single_ops = { - .atomic_write_len = PAGE_SIZE, - .open = cgroup_file_open, - .release = cgroup_file_release, - .write = cgroup_file_write, - .seq_show = cgroup_seqfile_show, -}; - -static struct kernfs_ops cgroup_kf_ops = { - .atomic_write_len = PAGE_SIZE, - .open = cgroup_file_open, - .release = cgroup_file_release, - .write = cgroup_file_write, - .seq_start = cgroup_seqfile_start, - .seq_next = cgroup_seqfile_next, - .seq_stop = cgroup_seqfile_stop, - .seq_show = cgroup_seqfile_show, -}; - -/* - * cgroup_rename - Only allow simple rename of directories in place. - */ -static int cgroup_rename(struct kernfs_node *kn, struct kernfs_node *new_parent, - const char *new_name_str) -{ - struct cgroup *cgrp = kn->priv; - int ret; - - if (kernfs_type(kn) != KERNFS_DIR) - return -ENOTDIR; - if (kn->parent != new_parent) - return -EIO; - - /* - * This isn't a proper migration and its usefulness is very - * limited. Disallow on the default hierarchy. - */ - if (cgroup_on_dfl(cgrp)) - return -EPERM; - - /* - * We're gonna grab cgroup_mutex which nests outside kernfs - * active_ref. kernfs_rename() doesn't require active_ref - * protection. Break them before grabbing cgroup_mutex. - */ - kernfs_break_active_protection(new_parent); - kernfs_break_active_protection(kn); - - mutex_lock(&cgroup_mutex); - - ret = kernfs_rename(kn, new_parent, new_name_str); - if (!ret) - trace_cgroup_rename(cgrp); - - mutex_unlock(&cgroup_mutex); - - kernfs_unbreak_active_protection(kn); - kernfs_unbreak_active_protection(new_parent); - return ret; -} - -/* set uid and gid of cgroup dirs and files to that of the creator */ -static int cgroup_kn_set_ugid(struct kernfs_node *kn) -{ - struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID, - .ia_uid = current_fsuid(), - .ia_gid = current_fsgid(), }; - - if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) && - gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID)) - return 0; - - return kernfs_setattr(kn, &iattr); -} - -static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp, - struct cftype *cft) -{ - char name[CGROUP_FILE_NAME_MAX]; - struct kernfs_node *kn; - struct lock_class_key *key = NULL; - int ret; - -#ifdef CONFIG_DEBUG_LOCK_ALLOC - key = &cft->lockdep_key; -#endif - kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name), - cgroup_file_mode(cft), 0, cft->kf_ops, cft, - NULL, key); - if (IS_ERR(kn)) - return PTR_ERR(kn); - - ret = cgroup_kn_set_ugid(kn); - if (ret) { - kernfs_remove(kn); - return ret; - } - - if (cft->file_offset) { - struct cgroup_file *cfile = (void *)css + cft->file_offset; - - spin_lock_irq(&cgroup_file_kn_lock); - cfile->kn = kn; - spin_unlock_irq(&cgroup_file_kn_lock); - } - - return 0; -} - -/** - * cgroup_addrm_files - add or remove files to a cgroup directory - * @css: the target css - * @cgrp: the target cgroup (usually css->cgroup) - * @cfts: array of cftypes to be added - * @is_add: whether to add or remove - * - * Depending on @is_add, add or remove files defined by @cfts on @cgrp. - * For removals, this function never fails. - */ -static int cgroup_addrm_files(struct cgroup_subsys_state *css, - struct cgroup *cgrp, struct cftype cfts[], - bool is_add) -{ - struct cftype *cft, *cft_end = NULL; - int ret = 0; - - lockdep_assert_held(&cgroup_mutex); - -restart: - for (cft = cfts; cft != cft_end && cft->name[0] != '\0'; cft++) { - /* does cft->flags tell us to skip this file on @cgrp? */ - if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp)) - continue; - if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp)) - continue; - if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp)) - continue; - if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp)) - continue; - - if (is_add) { - ret = cgroup_add_file(css, cgrp, cft); - if (ret) { - pr_warn("%s: failed to add %s, err=%d\n", - __func__, cft->name, ret); - cft_end = cft; - is_add = false; - goto restart; - } - } else { - cgroup_rm_file(cgrp, cft); - } - } - return ret; -} - -static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add) -{ - LIST_HEAD(pending); - struct cgroup_subsys *ss = cfts[0].ss; - struct cgroup *root = &ss->root->cgrp; - struct cgroup_subsys_state *css; - int ret = 0; - - lockdep_assert_held(&cgroup_mutex); - - /* add/rm files for all cgroups created before */ - css_for_each_descendant_pre(css, cgroup_css(root, ss)) { - struct cgroup *cgrp = css->cgroup; - - if (!(css->flags & CSS_VISIBLE)) - continue; - - ret = cgroup_addrm_files(css, cgrp, cfts, is_add); - if (ret) - break; - } - - if (is_add && !ret) - kernfs_activate(root->kn); - return ret; -} - -static void cgroup_exit_cftypes(struct cftype *cfts) -{ - struct cftype *cft; - - for (cft = cfts; cft->name[0] != '\0'; cft++) { - /* free copy for custom atomic_write_len, see init_cftypes() */ - if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) - kfree(cft->kf_ops); - cft->kf_ops = NULL; - cft->ss = NULL; - - /* revert flags set by cgroup core while adding @cfts */ - cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL); - } -} - -static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) -{ - struct cftype *cft; - - for (cft = cfts; cft->name[0] != '\0'; cft++) { - struct kernfs_ops *kf_ops; - - WARN_ON(cft->ss || cft->kf_ops); - - if (cft->seq_start) - kf_ops = &cgroup_kf_ops; - else - kf_ops = &cgroup_kf_single_ops; - - /* - * Ugh... if @cft wants a custom max_write_len, we need to - * make a copy of kf_ops to set its atomic_write_len. - */ - if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) { - kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL); - if (!kf_ops) { - cgroup_exit_cftypes(cfts); - return -ENOMEM; - } - kf_ops->atomic_write_len = cft->max_write_len; - } - - cft->kf_ops = kf_ops; - cft->ss = ss; - } - - return 0; -} - -static int cgroup_rm_cftypes_locked(struct cftype *cfts) -{ - lockdep_assert_held(&cgroup_mutex); - - if (!cfts || !cfts[0].ss) - return -ENOENT; - - list_del(&cfts->node); - cgroup_apply_cftypes(cfts, false); - cgroup_exit_cftypes(cfts); - return 0; -} - -/** - * cgroup_rm_cftypes - remove an array of cftypes from a subsystem - * @cfts: zero-length name terminated array of cftypes - * - * Unregister @cfts. Files described by @cfts are removed from all - * existing cgroups and all future cgroups won't have them either. This - * function can be called anytime whether @cfts' subsys is attached or not. - * - * Returns 0 on successful unregistration, -ENOENT if @cfts is not - * registered. - */ -int cgroup_rm_cftypes(struct cftype *cfts) -{ - int ret; - - mutex_lock(&cgroup_mutex); - ret = cgroup_rm_cftypes_locked(cfts); - mutex_unlock(&cgroup_mutex); - return ret; -} - -/** - * cgroup_add_cftypes - add an array of cftypes to a subsystem - * @ss: target cgroup subsystem - * @cfts: zero-length name terminated array of cftypes - * - * Register @cfts to @ss. Files described by @cfts are created for all - * existing cgroups to which @ss is attached and all future cgroups will - * have them too. This function can be called anytime whether @ss is - * attached or not. - * - * Returns 0 on successful registration, -errno on failure. Note that this - * function currently returns 0 as long as @cfts registration is successful - * even if some file creation attempts on existing cgroups fail. - */ -static int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) -{ - int ret; - - if (!cgroup_ssid_enabled(ss->id)) - return 0; - - if (!cfts || cfts[0].name[0] == '\0') - return 0; - - ret = cgroup_init_cftypes(ss, cfts); - if (ret) - return ret; - - mutex_lock(&cgroup_mutex); - - list_add_tail(&cfts->node, &ss->cfts); - ret = cgroup_apply_cftypes(cfts, true); - if (ret) - cgroup_rm_cftypes_locked(cfts); - - mutex_unlock(&cgroup_mutex); - return ret; -} - -/** - * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy - * @ss: target cgroup subsystem - * @cfts: zero-length name terminated array of cftypes - * - * Similar to cgroup_add_cftypes() but the added files are only used for - * the default hierarchy. - */ -int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) -{ - struct cftype *cft; - - for (cft = cfts; cft && cft->name[0] != '\0'; cft++) - cft->flags |= __CFTYPE_ONLY_ON_DFL; - return cgroup_add_cftypes(ss, cfts); -} - -/** - * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies - * @ss: target cgroup subsystem - * @cfts: zero-length name terminated array of cftypes - * - * Similar to cgroup_add_cftypes() but the added files are only used for - * the legacy hierarchies. - */ -int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) -{ - struct cftype *cft; - - for (cft = cfts; cft && cft->name[0] != '\0'; cft++) - cft->flags |= __CFTYPE_NOT_ON_DFL; - return cgroup_add_cftypes(ss, cfts); -} - -/** - * cgroup_file_notify - generate a file modified event for a cgroup_file - * @cfile: target cgroup_file - * - * @cfile must have been obtained by setting cftype->file_offset. - */ -void cgroup_file_notify(struct cgroup_file *cfile) -{ - unsigned long flags; - - spin_lock_irqsave(&cgroup_file_kn_lock, flags); - if (cfile->kn) - kernfs_notify(cfile->kn); - spin_unlock_irqrestore(&cgroup_file_kn_lock, flags); -} - -/** - * cgroup_task_count - count the number of tasks in a cgroup. - * @cgrp: the cgroup in question - * - * Return the number of tasks in the cgroup. The returned number can be - * higher than the actual number of tasks due to css_set references from - * namespace roots and temporary usages. - */ -static int cgroup_task_count(const struct cgroup *cgrp) -{ - int count = 0; - struct cgrp_cset_link *link; - - spin_lock_irq(&css_set_lock); - list_for_each_entry(link, &cgrp->cset_links, cset_link) - count += atomic_read(&link->cset->refcount); - spin_unlock_irq(&css_set_lock); - return count; -} - -/** - * css_next_child - find the next child of a given css - * @pos: the current position (%NULL to initiate traversal) - * @parent: css whose children to walk - * - * This function returns the next child of @parent and should be called - * under either cgroup_mutex or RCU read lock. The only requirement is - * that @parent and @pos are accessible. The next sibling is guaranteed to - * be returned regardless of their states. - * - * If a subsystem synchronizes ->css_online() and the start of iteration, a - * css which finished ->css_online() is guaranteed to be visible in the - * future iterations and will stay visible until the last reference is put. - * A css which hasn't finished ->css_online() or already finished - * ->css_offline() may show up during traversal. It's each subsystem's - * responsibility to synchronize against on/offlining. - */ -struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos, - struct cgroup_subsys_state *parent) -{ - struct cgroup_subsys_state *next; - - cgroup_assert_mutex_or_rcu_locked(); - - /* - * @pos could already have been unlinked from the sibling list. - * Once a cgroup is removed, its ->sibling.next is no longer - * updated when its next sibling changes. CSS_RELEASED is set when - * @pos is taken off list, at which time its next pointer is valid, - * and, as releases are serialized, the one pointed to by the next - * pointer is guaranteed to not have started release yet. This - * implies that if we observe !CSS_RELEASED on @pos in this RCU - * critical section, the one pointed to by its next pointer is - * guaranteed to not have finished its RCU grace period even if we - * have dropped rcu_read_lock() inbetween iterations. - * - * If @pos has CSS_RELEASED set, its next pointer can't be - * dereferenced; however, as each css is given a monotonically - * increasing unique serial number and always appended to the - * sibling list, the next one can be found by walking the parent's - * children until the first css with higher serial number than - * @pos's. While this path can be slower, it happens iff iteration - * races against release and the race window is very small. - */ - if (!pos) { - next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling); - } else if (likely(!(pos->flags & CSS_RELEASED))) { - next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling); - } else { - list_for_each_entry_rcu(next, &parent->children, sibling) - if (next->serial_nr > pos->serial_nr) - break; - } - - /* - * @next, if not pointing to the head, can be dereferenced and is - * the next sibling. - */ - if (&next->sibling != &parent->children) - return next; - return NULL; -} - -/** - * css_next_descendant_pre - find the next descendant for pre-order walk - * @pos: the current position (%NULL to initiate traversal) - * @root: css whose descendants to walk - * - * To be used by css_for_each_descendant_pre(). Find the next descendant - * to visit for pre-order traversal of @root's descendants. @root is - * included in the iteration and the first node to be visited. - * - * While this function requires cgroup_mutex or RCU read locking, it - * doesn't require the whole traversal to be contained in a single critical - * section. This function will return the correct next descendant as long - * as both @pos and @root are accessible and @pos is a descendant of @root. - * - * If a subsystem synchronizes ->css_online() and the start of iteration, a - * css which finished ->css_online() is guaranteed to be visible in the - * future iterations and will stay visible until the last reference is put. - * A css which hasn't finished ->css_online() or already finished - * ->css_offline() may show up during traversal. It's each subsystem's - * responsibility to synchronize against on/offlining. - */ -struct cgroup_subsys_state * -css_next_descendant_pre(struct cgroup_subsys_state *pos, - struct cgroup_subsys_state *root) -{ - struct cgroup_subsys_state *next; - - cgroup_assert_mutex_or_rcu_locked(); - - /* if first iteration, visit @root */ - if (!pos) - return root; - - /* visit the first child if exists */ - next = css_next_child(NULL, pos); - if (next) - return next; - - /* no child, visit my or the closest ancestor's next sibling */ - while (pos != root) { - next = css_next_child(pos, pos->parent); - if (next) - return next; - pos = pos->parent; - } - - return NULL; -} - -/** - * css_rightmost_descendant - return the rightmost descendant of a css - * @pos: css of interest - * - * Return the rightmost descendant of @pos. If there's no descendant, @pos - * is returned. This can be used during pre-order traversal to skip - * subtree of @pos. - * - * While this function requires cgroup_mutex or RCU read locking, it - * doesn't require the whole traversal to be contained in a single critical - * section. This function will return the correct rightmost descendant as - * long as @pos is accessible. - */ -struct cgroup_subsys_state * -css_rightmost_descendant(struct cgroup_subsys_state *pos) -{ - struct cgroup_subsys_state *last, *tmp; - - cgroup_assert_mutex_or_rcu_locked(); - - do { - last = pos; - /* ->prev isn't RCU safe, walk ->next till the end */ - pos = NULL; - css_for_each_child(tmp, last) - pos = tmp; - } while (pos); - - return last; -} - -static struct cgroup_subsys_state * -css_leftmost_descendant(struct cgroup_subsys_state *pos) -{ - struct cgroup_subsys_state *last; - - do { - last = pos; - pos = css_next_child(NULL, pos); - } while (pos); - - return last; -} - -/** - * css_next_descendant_post - find the next descendant for post-order walk - * @pos: the current position (%NULL to initiate traversal) - * @root: css whose descendants to walk - * - * To be used by css_for_each_descendant_post(). Find the next descendant - * to visit for post-order traversal of @root's descendants. @root is - * included in the iteration and the last node to be visited. - * - * While this function requires cgroup_mutex or RCU read locking, it - * doesn't require the whole traversal to be contained in a single critical - * section. This function will return the correct next descendant as long - * as both @pos and @cgroup are accessible and @pos is a descendant of - * @cgroup. - * - * If a subsystem synchronizes ->css_online() and the start of iteration, a - * css which finished ->css_online() is guaranteed to be visible in the - * future iterations and will stay visible until the last reference is put. - * A css which hasn't finished ->css_online() or already finished - * ->css_offline() may show up during traversal. It's each subsystem's - * responsibility to synchronize against on/offlining. - */ -struct cgroup_subsys_state * -css_next_descendant_post(struct cgroup_subsys_state *pos, - struct cgroup_subsys_state *root) -{ - struct cgroup_subsys_state *next; - - cgroup_assert_mutex_or_rcu_locked(); - - /* if first iteration, visit leftmost descendant which may be @root */ - if (!pos) - return css_leftmost_descendant(root); - - /* if we visited @root, we're done */ - if (pos == root) - return NULL; - - /* if there's an unvisited sibling, visit its leftmost descendant */ - next = css_next_child(pos, pos->parent); - if (next) - return css_leftmost_descendant(next); - - /* no sibling left, visit parent */ - return pos->parent; -} - -/** - * css_has_online_children - does a css have online children - * @css: the target css - * - * Returns %true if @css has any online children; otherwise, %false. This - * function can be called from any context but the caller is responsible - * for synchronizing against on/offlining as necessary. - */ -bool css_has_online_children(struct cgroup_subsys_state *css) -{ - struct cgroup_subsys_state *child; - bool ret = false; - - rcu_read_lock(); - css_for_each_child(child, css) { - if (child->flags & CSS_ONLINE) { - ret = true; - break; - } - } - rcu_read_unlock(); - return ret; -} - -/** - * css_task_iter_advance_css_set - advance a task itererator to the next css_set - * @it: the iterator to advance - * - * Advance @it to the next css_set to walk. - */ -static void css_task_iter_advance_css_set(struct css_task_iter *it) -{ - struct list_head *l = it->cset_pos; - struct cgrp_cset_link *link; - struct css_set *cset; - - lockdep_assert_held(&css_set_lock); - - /* Advance to the next non-empty css_set */ - do { - l = l->next; - if (l == it->cset_head) { - it->cset_pos = NULL; - it->task_pos = NULL; - return; - } - - if (it->ss) { - cset = container_of(l, struct css_set, - e_cset_node[it->ss->id]); - } else { - link = list_entry(l, struct cgrp_cset_link, cset_link); - cset = link->cset; - } - } while (!css_set_populated(cset)); - - it->cset_pos = l; - - if (!list_empty(&cset->tasks)) - it->task_pos = cset->tasks.next; - else - it->task_pos = cset->mg_tasks.next; - - it->tasks_head = &cset->tasks; - it->mg_tasks_head = &cset->mg_tasks; - - /* - * We don't keep css_sets locked across iteration steps and thus - * need to take steps to ensure that iteration can be resumed after - * the lock is re-acquired. Iteration is performed at two levels - - * css_sets and tasks in them. - * - * Once created, a css_set never leaves its cgroup lists, so a - * pinned css_set is guaranteed to stay put and we can resume - * iteration afterwards. - * - * Tasks may leave @cset across iteration steps. This is resolved - * by registering each iterator with the css_set currently being - * walked and making css_set_move_task() advance iterators whose - * next task is leaving. - */ - if (it->cur_cset) { - list_del(&it->iters_node); - put_css_set_locked(it->cur_cset); - } - get_css_set(cset); - it->cur_cset = cset; - list_add(&it->iters_node, &cset->task_iters); -} - -static void css_task_iter_advance(struct css_task_iter *it) -{ - struct list_head *l = it->task_pos; - - lockdep_assert_held(&css_set_lock); - WARN_ON_ONCE(!l); - - /* - * Advance iterator to find next entry. cset->tasks is consumed - * first and then ->mg_tasks. After ->mg_tasks, we move onto the - * next cset. - */ - l = l->next; - - if (l == it->tasks_head) - l = it->mg_tasks_head->next; - - if (l == it->mg_tasks_head) - css_task_iter_advance_css_set(it); - else - it->task_pos = l; -} - -/** - * css_task_iter_start - initiate task iteration - * @css: the css to walk tasks of - * @it: the task iterator to use - * - * Initiate iteration through the tasks of @css. The caller can call - * css_task_iter_next() to walk through the tasks until the function - * returns NULL. On completion of iteration, css_task_iter_end() must be - * called. - */ -void css_task_iter_start(struct cgroup_subsys_state *css, - struct css_task_iter *it) -{ - /* no one should try to iterate before mounting cgroups */ - WARN_ON_ONCE(!use_task_css_set_links); - - memset(it, 0, sizeof(*it)); - - spin_lock_irq(&css_set_lock); - - it->ss = css->ss; - - if (it->ss) - it->cset_pos = &css->cgroup->e_csets[css->ss->id]; - else - it->cset_pos = &css->cgroup->cset_links; - - it->cset_head = it->cset_pos; - - css_task_iter_advance_css_set(it); - - spin_unlock_irq(&css_set_lock); -} - -/** - * css_task_iter_next - return the next task for the iterator - * @it: the task iterator being iterated - * - * The "next" function for task iteration. @it should have been - * initialized via css_task_iter_start(). Returns NULL when the iteration - * reaches the end. - */ -struct task_struct *css_task_iter_next(struct css_task_iter *it) -{ - if (it->cur_task) { - put_task_struct(it->cur_task); - it->cur_task = NULL; - } - - spin_lock_irq(&css_set_lock); - - if (it->task_pos) { - it->cur_task = list_entry(it->task_pos, struct task_struct, - cg_list); - get_task_struct(it->cur_task); - css_task_iter_advance(it); - } - - spin_unlock_irq(&css_set_lock); - - return it->cur_task; -} - -/** - * css_task_iter_end - finish task iteration - * @it: the task iterator to finish - * - * Finish task iteration started by css_task_iter_start(). - */ -void css_task_iter_end(struct css_task_iter *it) -{ - if (it->cur_cset) { - spin_lock_irq(&css_set_lock); - list_del(&it->iters_node); - put_css_set_locked(it->cur_cset); - spin_unlock_irq(&css_set_lock); - } - - if (it->cur_task) - put_task_struct(it->cur_task); -} - -/** - * cgroup_trasnsfer_tasks - move tasks from one cgroup to another - * @to: cgroup to which the tasks will be moved - * @from: cgroup in which the tasks currently reside - * - * Locking rules between cgroup_post_fork() and the migration path - * guarantee that, if a task is forking while being migrated, the new child - * is guaranteed to be either visible in the source cgroup after the - * parent's migration is complete or put into the target cgroup. No task - * can slip out of migration through forking. - */ -int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from) -{ - LIST_HEAD(preloaded_csets); - struct cgrp_cset_link *link; - struct css_task_iter it; - struct task_struct *task; - int ret; - - if (cgroup_on_dfl(to)) - return -EINVAL; - - if (!cgroup_may_migrate_to(to)) - return -EBUSY; - - mutex_lock(&cgroup_mutex); - - percpu_down_write(&cgroup_threadgroup_rwsem); - - /* all tasks in @from are being moved, all csets are source */ - spin_lock_irq(&css_set_lock); - list_for_each_entry(link, &from->cset_links, cset_link) - cgroup_migrate_add_src(link->cset, to, &preloaded_csets); - spin_unlock_irq(&css_set_lock); - - ret = cgroup_migrate_prepare_dst(&preloaded_csets); - if (ret) - goto out_err; - - /* - * Migrate tasks one-by-one until @from is empty. This fails iff - * ->can_attach() fails. - */ - do { - css_task_iter_start(&from->self, &it); - task = css_task_iter_next(&it); - if (task) - get_task_struct(task); - css_task_iter_end(&it); - - if (task) { - ret = cgroup_migrate(task, false, to->root); - if (!ret) - trace_cgroup_transfer_tasks(to, task, false); - put_task_struct(task); - } - } while (task && !ret); -out_err: - cgroup_migrate_finish(&preloaded_csets); - percpu_up_write(&cgroup_threadgroup_rwsem); - mutex_unlock(&cgroup_mutex); - return ret; -} - -static void cgroup_procs_release(struct kernfs_open_file *of) -{ - if (of->priv) { - css_task_iter_end(of->priv); - kfree(of->priv); - } -} - -static void *cgroup_procs_next(struct seq_file *s, void *v, loff_t *pos) -{ - struct kernfs_open_file *of = s->private; - struct css_task_iter *it = of->priv; - struct task_struct *task; - - do { - task = css_task_iter_next(it); - } while (task && !thread_group_leader(task)); - - return task; -} - -static void *cgroup_procs_start(struct seq_file *s, loff_t *pos) -{ - struct kernfs_open_file *of = s->private; - struct cgroup *cgrp = seq_css(s)->cgroup; - struct css_task_iter *it = of->priv; - - /* - * When a seq_file is seeked, it's always traversed sequentially - * from position 0, so we can simply keep iterating on !0 *pos. - */ - if (!it) { - if (WARN_ON_ONCE((*pos)++)) - return ERR_PTR(-EINVAL); - - it = kzalloc(sizeof(*it), GFP_KERNEL); - if (!it) - return ERR_PTR(-ENOMEM); - of->priv = it; - css_task_iter_start(&cgrp->self, it); - } else if (!(*pos)++) { - css_task_iter_end(it); - css_task_iter_start(&cgrp->self, it); - } - - return cgroup_procs_next(s, NULL, NULL); -} - -static int cgroup_procs_show(struct seq_file *s, void *v) -{ - seq_printf(s, "%d\n", task_tgid_vnr(v)); - return 0; -} - -/* - * Stuff for reading the 'tasks'/'procs' files. - * - * Reading this file can return large amounts of data if a cgroup has - * *lots* of attached tasks. So it may need several calls to read(), - * but we cannot guarantee that the information we produce is correct - * unless we produce it entirely atomically. - * - */ - -/* which pidlist file are we talking about? */ -enum cgroup_filetype { - CGROUP_FILE_PROCS, - CGROUP_FILE_TASKS, -}; - -/* - * A pidlist is a list of pids that virtually represents the contents of one - * of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists, - * a pair (one each for procs, tasks) for each pid namespace that's relevant - * to the cgroup. - */ -struct cgroup_pidlist { - /* - * used to find which pidlist is wanted. doesn't change as long as - * this particular list stays in the list. - */ - struct { enum cgroup_filetype type; struct pid_namespace *ns; } key; - /* array of xids */ - pid_t *list; - /* how many elements the above list has */ - int length; - /* each of these stored in a list by its cgroup */ - struct list_head links; - /* pointer to the cgroup we belong to, for list removal purposes */ - struct cgroup *owner; - /* for delayed destruction */ - struct delayed_work destroy_dwork; -}; - -/* - * The following two functions "fix" the issue where there are more pids - * than kmalloc will give memory for; in such cases, we use vmalloc/vfree. - * TODO: replace with a kernel-wide solution to this problem - */ -#define PIDLIST_TOO_LARGE(c) ((c) * sizeof(pid_t) > (PAGE_SIZE * 2)) -static void *pidlist_allocate(int count) -{ - if (PIDLIST_TOO_LARGE(count)) - return vmalloc(count * sizeof(pid_t)); - else - return kmalloc(count * sizeof(pid_t), GFP_KERNEL); -} - -static void pidlist_free(void *p) -{ - kvfree(p); -} - -/* - * Used to destroy all pidlists lingering waiting for destroy timer. None - * should be left afterwards. - */ -static void cgroup_pidlist_destroy_all(struct cgroup *cgrp) -{ - struct cgroup_pidlist *l, *tmp_l; - - mutex_lock(&cgrp->pidlist_mutex); - list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links) - mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0); - mutex_unlock(&cgrp->pidlist_mutex); - - flush_workqueue(cgroup_pidlist_destroy_wq); - BUG_ON(!list_empty(&cgrp->pidlists)); -} - -static void cgroup_pidlist_destroy_work_fn(struct work_struct *work) -{ - struct delayed_work *dwork = to_delayed_work(work); - struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist, - destroy_dwork); - struct cgroup_pidlist *tofree = NULL; - - mutex_lock(&l->owner->pidlist_mutex); - - /* - * Destroy iff we didn't get queued again. The state won't change - * as destroy_dwork can only be queued while locked. - */ - if (!delayed_work_pending(dwork)) { - list_del(&l->links); - pidlist_free(l->list); - put_pid_ns(l->key.ns); - tofree = l; - } - - mutex_unlock(&l->owner->pidlist_mutex); - kfree(tofree); -} - -/* - * pidlist_uniq - given a kmalloc()ed list, strip out all duplicate entries - * Returns the number of unique elements. - */ -static int pidlist_uniq(pid_t *list, int length) -{ - int src, dest = 1; - - /* - * we presume the 0th element is unique, so i starts at 1. trivial - * edge cases first; no work needs to be done for either - */ - if (length == 0 || length == 1) - return length; - /* src and dest walk down the list; dest counts unique elements */ - for (src = 1; src < length; src++) { - /* find next unique element */ - while (list[src] == list[src-1]) { - src++; - if (src == length) - goto after; - } - /* dest always points to where the next unique element goes */ - list[dest] = list[src]; - dest++; - } -after: - return dest; -} - -/* - * The two pid files - task and cgroup.procs - guaranteed that the result - * is sorted, which forced this whole pidlist fiasco. As pid order is - * different per namespace, each namespace needs differently sorted list, - * making it impossible to use, for example, single rbtree of member tasks - * sorted by task pointer. As pidlists can be fairly large, allocating one - * per open file is dangerous, so cgroup had to implement shared pool of - * pidlists keyed by cgroup and namespace. - */ -static int cmppid(const void *a, const void *b) -{ - return *(pid_t *)a - *(pid_t *)b; -} - -static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp, - enum cgroup_filetype type) -{ - struct cgroup_pidlist *l; - /* don't need task_nsproxy() if we're looking at ourself */ - struct pid_namespace *ns = task_active_pid_ns(current); - - lockdep_assert_held(&cgrp->pidlist_mutex); - - list_for_each_entry(l, &cgrp->pidlists, links) - if (l->key.type == type && l->key.ns == ns) - return l; - return NULL; -} - -/* - * find the appropriate pidlist for our purpose (given procs vs tasks) - * returns with the lock on that pidlist already held, and takes care - * of the use count, or returns NULL with no locks held if we're out of - * memory. - */ -static struct cgroup_pidlist *cgroup_pidlist_find_create(struct cgroup *cgrp, - enum cgroup_filetype type) -{ - struct cgroup_pidlist *l; - - lockdep_assert_held(&cgrp->pidlist_mutex); - - l = cgroup_pidlist_find(cgrp, type); - if (l) - return l; - - /* entry not found; create a new one */ - l = kzalloc(sizeof(struct cgroup_pidlist), GFP_KERNEL); - if (!l) - return l; - - INIT_DELAYED_WORK(&l->destroy_dwork, cgroup_pidlist_destroy_work_fn); - l->key.type = type; - /* don't need task_nsproxy() if we're looking at ourself */ - l->key.ns = get_pid_ns(task_active_pid_ns(current)); - l->owner = cgrp; - list_add(&l->links, &cgrp->pidlists); - return l; -} - -/* - * Load a cgroup's pidarray with either procs' tgids or tasks' pids - */ -static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type, - struct cgroup_pidlist **lp) -{ - pid_t *array; - int length; - int pid, n = 0; /* used for populating the array */ - struct css_task_iter it; - struct task_struct *tsk; - struct cgroup_pidlist *l; - - lockdep_assert_held(&cgrp->pidlist_mutex); - - /* - * If cgroup gets more users after we read count, we won't have - * enough space - tough. This race is indistinguishable to the - * caller from the case that the additional cgroup users didn't - * show up until sometime later on. - */ - length = cgroup_task_count(cgrp); - array = pidlist_allocate(length); - if (!array) - return -ENOMEM; - /* now, populate the array */ - css_task_iter_start(&cgrp->self, &it); - while ((tsk = css_task_iter_next(&it))) { - if (unlikely(n == length)) - break; - /* get tgid or pid for procs or tasks file respectively */ - if (type == CGROUP_FILE_PROCS) - pid = task_tgid_vnr(tsk); - else - pid = task_pid_vnr(tsk); - if (pid > 0) /* make sure to only use valid results */ - array[n++] = pid; - } - css_task_iter_end(&it); - length = n; - /* now sort & (if procs) strip out duplicates */ - sort(array, length, sizeof(pid_t), cmppid, NULL); - if (type == CGROUP_FILE_PROCS) - length = pidlist_uniq(array, length); - - l = cgroup_pidlist_find_create(cgrp, type); - if (!l) { - pidlist_free(array); - return -ENOMEM; - } - - /* store array, freeing old if necessary */ - pidlist_free(l->list); - l->list = array; - l->length = length; - *lp = l; - return 0; -} - -/** - * cgroupstats_build - build and fill cgroupstats - * @stats: cgroupstats to fill information into - * @dentry: A dentry entry belonging to the cgroup for which stats have - * been requested. - * - * Build and fill cgroupstats so that taskstats can export it to user - * space. - */ -int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry) -{ - struct kernfs_node *kn = kernfs_node_from_dentry(dentry); - struct cgroup *cgrp; - struct css_task_iter it; - struct task_struct *tsk; - - /* it should be kernfs_node belonging to cgroupfs and is a directory */ - if (dentry->d_sb->s_type != &cgroup_fs_type || !kn || - kernfs_type(kn) != KERNFS_DIR) - return -EINVAL; - - mutex_lock(&cgroup_mutex); - - /* - * We aren't being called from kernfs and there's no guarantee on - * @kn->priv's validity. For this and css_tryget_online_from_dir(), - * @kn->priv is RCU safe. Let's do the RCU dancing. - */ - rcu_read_lock(); - cgrp = rcu_dereference(kn->priv); - if (!cgrp || cgroup_is_dead(cgrp)) { - rcu_read_unlock(); - mutex_unlock(&cgroup_mutex); - return -ENOENT; - } - rcu_read_unlock(); - - css_task_iter_start(&cgrp->self, &it); - while ((tsk = css_task_iter_next(&it))) { - switch (tsk->state) { - case TASK_RUNNING: - stats->nr_running++; - break; - case TASK_INTERRUPTIBLE: - stats->nr_sleeping++; - break; - case TASK_UNINTERRUPTIBLE: - stats->nr_uninterruptible++; - break; - case TASK_STOPPED: - stats->nr_stopped++; - break; - default: - if (delayacct_is_task_waiting_on_io(tsk)) - stats->nr_io_wait++; - break; - } - } - css_task_iter_end(&it); - - mutex_unlock(&cgroup_mutex); - return 0; -} - - -/* - * seq_file methods for the tasks/procs files. The seq_file position is the - * next pid to display; the seq_file iterator is a pointer to the pid - * in the cgroup->l->list array. - */ - -static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos) -{ - /* - * Initially we receive a position value that corresponds to - * one more than the last pid shown (or 0 on the first call or - * after a seek to the start). Use a binary-search to find the - * next pid to display, if any - */ - struct kernfs_open_file *of = s->private; - struct cgroup *cgrp = seq_css(s)->cgroup; - struct cgroup_pidlist *l; - enum cgroup_filetype type = seq_cft(s)->private; - int index = 0, pid = *pos; - int *iter, ret; - - mutex_lock(&cgrp->pidlist_mutex); - - /* - * !NULL @of->priv indicates that this isn't the first start() - * after open. If the matching pidlist is around, we can use that. - * Look for it. Note that @of->priv can't be used directly. It - * could already have been destroyed. - */ - if (of->priv) - of->priv = cgroup_pidlist_find(cgrp, type); - - /* - * Either this is the first start() after open or the matching - * pidlist has been destroyed inbetween. Create a new one. - */ - if (!of->priv) { - ret = pidlist_array_load(cgrp, type, - (struct cgroup_pidlist **)&of->priv); - if (ret) - return ERR_PTR(ret); - } - l = of->priv; - - if (pid) { - int end = l->length; - - while (index < end) { - int mid = (index + end) / 2; - if (l->list[mid] == pid) { - index = mid; - break; - } else if (l->list[mid] <= pid) - index = mid + 1; - else - end = mid; - } - } - /* If we're off the end of the array, we're done */ - if (index >= l->length) - return NULL; - /* Update the abstract position to be the actual pid that we found */ - iter = l->list + index; - *pos = *iter; - return iter; -} - -static void cgroup_pidlist_stop(struct seq_file *s, void *v) -{ - struct kernfs_open_file *of = s->private; - struct cgroup_pidlist *l = of->priv; - - if (l) - mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, - CGROUP_PIDLIST_DESTROY_DELAY); - mutex_unlock(&seq_css(s)->cgroup->pidlist_mutex); -} - -static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos) -{ - struct kernfs_open_file *of = s->private; - struct cgroup_pidlist *l = of->priv; - pid_t *p = v; - pid_t *end = l->list + l->length; - /* - * Advance to the next pid in the array. If this goes off the - * end, we're done - */ - p++; - if (p >= end) { - return NULL; - } else { - *pos = *p; - return p; - } -} - -static int cgroup_pidlist_show(struct seq_file *s, void *v) -{ - seq_printf(s, "%d\n", *(int *)v); - - return 0; -} - -static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css, - struct cftype *cft) -{ - return notify_on_release(css->cgroup); -} - -static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css, - struct cftype *cft, u64 val) -{ - if (val) - set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags); - else - clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags); - return 0; -} - -static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css, - struct cftype *cft) -{ - return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); -} - -static int cgroup_clone_children_write(struct cgroup_subsys_state *css, - struct cftype *cft, u64 val) -{ - if (val) - set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); - else - clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); - return 0; -} - -/* cgroup core interface files for the default hierarchy */ -static struct cftype cgroup_dfl_base_files[] = { - { - .name = "cgroup.procs", - .file_offset = offsetof(struct cgroup, procs_file), - .release = cgroup_procs_release, - .seq_start = cgroup_procs_start, - .seq_next = cgroup_procs_next, - .seq_show = cgroup_procs_show, - .write = cgroup_procs_write, - }, - { - .name = "cgroup.controllers", - .seq_show = cgroup_controllers_show, - }, - { - .name = "cgroup.subtree_control", - .seq_show = cgroup_subtree_control_show, - .write = cgroup_subtree_control_write, - }, - { - .name = "cgroup.events", - .flags = CFTYPE_NOT_ON_ROOT, - .file_offset = offsetof(struct cgroup, events_file), - .seq_show = cgroup_events_show, - }, - { } /* terminate */ -}; - -/* cgroup core interface files for the legacy hierarchies */ -static struct cftype cgroup_legacy_base_files[] = { - { - .name = "cgroup.procs", - .seq_start = cgroup_pidlist_start, - .seq_next = cgroup_pidlist_next, - .seq_stop = cgroup_pidlist_stop, - .seq_show = cgroup_pidlist_show, - .private = CGROUP_FILE_PROCS, - .write = cgroup_procs_write, - }, - { - .name = "cgroup.clone_children", - .read_u64 = cgroup_clone_children_read, - .write_u64 = cgroup_clone_children_write, - }, - { - .name = "cgroup.sane_behavior", - .flags = CFTYPE_ONLY_ON_ROOT, - .seq_show = cgroup_sane_behavior_show, - }, - { - .name = "tasks", - .seq_start = cgroup_pidlist_start, - .seq_next = cgroup_pidlist_next, - .seq_stop = cgroup_pidlist_stop, - .seq_show = cgroup_pidlist_show, - .private = CGROUP_FILE_TASKS, - .write = cgroup_tasks_write, - }, - { - .name = "notify_on_release", - .read_u64 = cgroup_read_notify_on_release, - .write_u64 = cgroup_write_notify_on_release, - }, - { - .name = "release_agent", - .flags = CFTYPE_ONLY_ON_ROOT, - .seq_show = cgroup_release_agent_show, - .write = cgroup_release_agent_write, - .max_write_len = PATH_MAX - 1, - }, - { } /* terminate */ -}; - -/* - * css destruction is four-stage process. - * - * 1. Destruction starts. Killing of the percpu_ref is initiated. - * Implemented in kill_css(). - * - * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs - * and thus css_tryget_online() is guaranteed to fail, the css can be - * offlined by invoking offline_css(). After offlining, the base ref is - * put. Implemented in css_killed_work_fn(). - * - * 3. When the percpu_ref reaches zero, the only possible remaining - * accessors are inside RCU read sections. css_release() schedules the - * RCU callback. - * - * 4. After the grace period, the css can be freed. Implemented in - * css_free_work_fn(). - * - * It is actually hairier because both step 2 and 4 require process context - * and thus involve punting to css->destroy_work adding two additional - * steps to the already complex sequence. - */ -static void css_free_work_fn(struct work_struct *work) -{ - struct cgroup_subsys_state *css = - container_of(work, struct cgroup_subsys_state, destroy_work); - struct cgroup_subsys *ss = css->ss; - struct cgroup *cgrp = css->cgroup; - - percpu_ref_exit(&css->refcnt); - - if (ss) { - /* css free path */ - struct cgroup_subsys_state *parent = css->parent; - int id = css->id; - - ss->css_free(css); - cgroup_idr_remove(&ss->css_idr, id); - cgroup_put(cgrp); - - if (parent) - css_put(parent); - } else { - /* cgroup free path */ - atomic_dec(&cgrp->root->nr_cgrps); - cgroup_pidlist_destroy_all(cgrp); - cancel_work_sync(&cgrp->release_agent_work); - - if (cgroup_parent(cgrp)) { - /* - * We get a ref to the parent, and put the ref when - * this cgroup is being freed, so it's guaranteed - * that the parent won't be destroyed before its - * children. - */ - cgroup_put(cgroup_parent(cgrp)); - kernfs_put(cgrp->kn); - kfree(cgrp); - } else { - /* - * This is root cgroup's refcnt reaching zero, - * which indicates that the root should be - * released. - */ - cgroup_destroy_root(cgrp->root); - } - } -} - -static void css_free_rcu_fn(struct rcu_head *rcu_head) -{ - struct cgroup_subsys_state *css = - container_of(rcu_head, struct cgroup_subsys_state, rcu_head); - - INIT_WORK(&css->destroy_work, css_free_work_fn); - queue_work(cgroup_destroy_wq, &css->destroy_work); -} - -static void css_release_work_fn(struct work_struct *work) -{ - struct cgroup_subsys_state *css = - container_of(work, struct cgroup_subsys_state, destroy_work); - struct cgroup_subsys *ss = css->ss; - struct cgroup *cgrp = css->cgroup; - - mutex_lock(&cgroup_mutex); - - css->flags |= CSS_RELEASED; - list_del_rcu(&css->sibling); - - if (ss) { - /* css release path */ - cgroup_idr_replace(&ss->css_idr, NULL, css->id); - if (ss->css_released) - ss->css_released(css); - } else { - /* cgroup release path */ - trace_cgroup_release(cgrp); - - cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id); - cgrp->id = -1; - - /* - * There are two control paths which try to determine - * cgroup from dentry without going through kernfs - - * cgroupstats_build() and css_tryget_online_from_dir(). - * Those are supported by RCU protecting clearing of - * cgrp->kn->priv backpointer. - */ - if (cgrp->kn) - RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv, - NULL); - - cgroup_bpf_put(cgrp); - } - - mutex_unlock(&cgroup_mutex); - - call_rcu(&css->rcu_head, css_free_rcu_fn); -} - -static void css_release(struct percpu_ref *ref) -{ - struct cgroup_subsys_state *css = - container_of(ref, struct cgroup_subsys_state, refcnt); - - INIT_WORK(&css->destroy_work, css_release_work_fn); - queue_work(cgroup_destroy_wq, &css->destroy_work); -} - -static void init_and_link_css(struct cgroup_subsys_state *css, - struct cgroup_subsys *ss, struct cgroup *cgrp) -{ - lockdep_assert_held(&cgroup_mutex); - - cgroup_get(cgrp); - - memset(css, 0, sizeof(*css)); - css->cgroup = cgrp; - css->ss = ss; - css->id = -1; - INIT_LIST_HEAD(&css->sibling); - INIT_LIST_HEAD(&css->children); - css->serial_nr = css_serial_nr_next++; - atomic_set(&css->online_cnt, 0); - - if (cgroup_parent(cgrp)) { - css->parent = cgroup_css(cgroup_parent(cgrp), ss); - css_get(css->parent); - } - - BUG_ON(cgroup_css(cgrp, ss)); -} - -/* invoke ->css_online() on a new CSS and mark it online if successful */ -static int online_css(struct cgroup_subsys_state *css) -{ - struct cgroup_subsys *ss = css->ss; - int ret = 0; - - lockdep_assert_held(&cgroup_mutex); - - if (ss->css_online) - ret = ss->css_online(css); - if (!ret) { - css->flags |= CSS_ONLINE; - rcu_assign_pointer(css->cgroup->subsys[ss->id], css); - - atomic_inc(&css->online_cnt); - if (css->parent) - atomic_inc(&css->parent->online_cnt); - } - return ret; -} - -/* if the CSS is online, invoke ->css_offline() on it and mark it offline */ -static void offline_css(struct cgroup_subsys_state *css) -{ - struct cgroup_subsys *ss = css->ss; - - lockdep_assert_held(&cgroup_mutex); - - if (!(css->flags & CSS_ONLINE)) - return; - - if (ss->css_reset) - ss->css_reset(css); - - if (ss->css_offline) - ss->css_offline(css); - - css->flags &= ~CSS_ONLINE; - RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL); - - wake_up_all(&css->cgroup->offline_waitq); -} - -/** - * css_create - create a cgroup_subsys_state - * @cgrp: the cgroup new css will be associated with - * @ss: the subsys of new css - * - * Create a new css associated with @cgrp - @ss pair. On success, the new - * css is online and installed in @cgrp. This function doesn't create the - * interface files. Returns 0 on success, -errno on failure. - */ -static struct cgroup_subsys_state *css_create(struct cgroup *cgrp, - struct cgroup_subsys *ss) -{ - struct cgroup *parent = cgroup_parent(cgrp); - struct cgroup_subsys_state *parent_css = cgroup_css(parent, ss); - struct cgroup_subsys_state *css; - int err; - - lockdep_assert_held(&cgroup_mutex); - - css = ss->css_alloc(parent_css); - if (!css) - css = ERR_PTR(-ENOMEM); - if (IS_ERR(css)) - return css; - - init_and_link_css(css, ss, cgrp); - - err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL); - if (err) - goto err_free_css; - - err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_KERNEL); - if (err < 0) - goto err_free_css; - css->id = err; - - /* @css is ready to be brought online now, make it visible */ - list_add_tail_rcu(&css->sibling, &parent_css->children); - cgroup_idr_replace(&ss->css_idr, css, css->id); - - err = online_css(css); - if (err) - goto err_list_del; - - if (ss->broken_hierarchy && !ss->warned_broken_hierarchy && - cgroup_parent(parent)) { - pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n", - current->comm, current->pid, ss->name); - if (!strcmp(ss->name, "memory")) - pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n"); - ss->warned_broken_hierarchy = true; - } - - return css; - -err_list_del: - list_del_rcu(&css->sibling); -err_free_css: - call_rcu(&css->rcu_head, css_free_rcu_fn); - return ERR_PTR(err); -} - -static struct cgroup *cgroup_create(struct cgroup *parent) -{ - struct cgroup_root *root = parent->root; - struct cgroup *cgrp, *tcgrp; - int level = parent->level + 1; - int ret; - - /* allocate the cgroup and its ID, 0 is reserved for the root */ - cgrp = kzalloc(sizeof(*cgrp) + - sizeof(cgrp->ancestor_ids[0]) * (level + 1), GFP_KERNEL); - if (!cgrp) - return ERR_PTR(-ENOMEM); - - ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL); - if (ret) - goto out_free_cgrp; - - /* - * Temporarily set the pointer to NULL, so idr_find() won't return - * a half-baked cgroup. - */ - cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_KERNEL); - if (cgrp->id < 0) { - ret = -ENOMEM; - goto out_cancel_ref; - } - - init_cgroup_housekeeping(cgrp); - - cgrp->self.parent = &parent->self; - cgrp->root = root; - cgrp->level = level; - - for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp)) - cgrp->ancestor_ids[tcgrp->level] = tcgrp->id; - - if (notify_on_release(parent)) - set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags); - - if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags)) - set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags); - - cgrp->self.serial_nr = css_serial_nr_next++; - - /* allocation complete, commit to creation */ - list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children); - atomic_inc(&root->nr_cgrps); - cgroup_get(parent); - - /* - * @cgrp is now fully operational. If something fails after this - * point, it'll be released via the normal destruction path. - */ - cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id); - - /* - * On the default hierarchy, a child doesn't automatically inherit - * subtree_control from the parent. Each is configured manually. - */ - if (!cgroup_on_dfl(cgrp)) - cgrp->subtree_control = cgroup_control(cgrp); - - if (parent) - cgroup_bpf_inherit(cgrp, parent); - - cgroup_propagate_control(cgrp); - - /* @cgrp doesn't have dir yet so the following will only create csses */ - ret = cgroup_apply_control_enable(cgrp); - if (ret) - goto out_destroy; - - return cgrp; - -out_cancel_ref: - percpu_ref_exit(&cgrp->self.refcnt); -out_free_cgrp: - kfree(cgrp); - return ERR_PTR(ret); -out_destroy: - cgroup_destroy_locked(cgrp); - return ERR_PTR(ret); -} - -static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, - umode_t mode) -{ - struct cgroup *parent, *cgrp; - struct kernfs_node *kn; - int ret; - - /* do not accept '\n' to prevent making /proc//cgroup unparsable */ - if (strchr(name, '\n')) - return -EINVAL; - - parent = cgroup_kn_lock_live(parent_kn, false); - if (!parent) - return -ENODEV; - - cgrp = cgroup_create(parent); - if (IS_ERR(cgrp)) { - ret = PTR_ERR(cgrp); - goto out_unlock; - } - - /* create the directory */ - kn = kernfs_create_dir(parent->kn, name, mode, cgrp); - if (IS_ERR(kn)) { - ret = PTR_ERR(kn); - goto out_destroy; - } - cgrp->kn = kn; - - /* - * This extra ref will be put in cgroup_free_fn() and guarantees - * that @cgrp->kn is always accessible. - */ - kernfs_get(kn); - - ret = cgroup_kn_set_ugid(kn); - if (ret) - goto out_destroy; - - ret = css_populate_dir(&cgrp->self); - if (ret) - goto out_destroy; - - ret = cgroup_apply_control_enable(cgrp); - if (ret) - goto out_destroy; - - trace_cgroup_mkdir(cgrp); - - /* let's create and online css's */ - kernfs_activate(kn); - - ret = 0; - goto out_unlock; - -out_destroy: - cgroup_destroy_locked(cgrp); -out_unlock: - cgroup_kn_unlock(parent_kn); - return ret; -} - -/* - * This is called when the refcnt of a css is confirmed to be killed. - * css_tryget_online() is now guaranteed to fail. Tell the subsystem to - * initate destruction and put the css ref from kill_css(). - */ -static void css_killed_work_fn(struct work_struct *work) -{ - struct cgroup_subsys_state *css = - container_of(work, struct cgroup_subsys_state, destroy_work); - - mutex_lock(&cgroup_mutex); - - do { - offline_css(css); - css_put(css); - /* @css can't go away while we're holding cgroup_mutex */ - css = css->parent; - } while (css && atomic_dec_and_test(&css->online_cnt)); - - mutex_unlock(&cgroup_mutex); -} - -/* css kill confirmation processing requires process context, bounce */ -static void css_killed_ref_fn(struct percpu_ref *ref) -{ - struct cgroup_subsys_state *css = - container_of(ref, struct cgroup_subsys_state, refcnt); - - if (atomic_dec_and_test(&css->online_cnt)) { - INIT_WORK(&css->destroy_work, css_killed_work_fn); - queue_work(cgroup_destroy_wq, &css->destroy_work); - } -} - -/** - * kill_css - destroy a css - * @css: css to destroy - * - * This function initiates destruction of @css by removing cgroup interface - * files and putting its base reference. ->css_offline() will be invoked - * asynchronously once css_tryget_online() is guaranteed to fail and when - * the reference count reaches zero, @css will be released. - */ -static void kill_css(struct cgroup_subsys_state *css) -{ - lockdep_assert_held(&cgroup_mutex); - - /* - * This must happen before css is disassociated with its cgroup. - * See seq_css() for details. - */ - css_clear_dir(css); - - /* - * Killing would put the base ref, but we need to keep it alive - * until after ->css_offline(). - */ - css_get(css); - - /* - * cgroup core guarantees that, by the time ->css_offline() is - * invoked, no new css reference will be given out via - * css_tryget_online(). We can't simply call percpu_ref_kill() and - * proceed to offlining css's because percpu_ref_kill() doesn't - * guarantee that the ref is seen as killed on all CPUs on return. - * - * Use percpu_ref_kill_and_confirm() to get notifications as each - * css is confirmed to be seen as killed on all CPUs. - */ - percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn); -} - -/** - * cgroup_destroy_locked - the first stage of cgroup destruction - * @cgrp: cgroup to be destroyed - * - * css's make use of percpu refcnts whose killing latency shouldn't be - * exposed to userland and are RCU protected. Also, cgroup core needs to - * guarantee that css_tryget_online() won't succeed by the time - * ->css_offline() is invoked. To satisfy all the requirements, - * destruction is implemented in the following two steps. - * - * s1. Verify @cgrp can be destroyed and mark it dying. Remove all - * userland visible parts and start killing the percpu refcnts of - * css's. Set up so that the next stage will be kicked off once all - * the percpu refcnts are confirmed to be killed. - * - * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the - * rest of destruction. Once all cgroup references are gone, the - * cgroup is RCU-freed. - * - * This function implements s1. After this step, @cgrp is gone as far as - * the userland is concerned and a new cgroup with the same name may be - * created. As cgroup doesn't care about the names internally, this - * doesn't cause any problem. - */ -static int cgroup_destroy_locked(struct cgroup *cgrp) - __releases(&cgroup_mutex) __acquires(&cgroup_mutex) -{ - struct cgroup_subsys_state *css; - struct cgrp_cset_link *link; - int ssid; - - lockdep_assert_held(&cgroup_mutex); - - /* - * Only migration can raise populated from zero and we're already - * holding cgroup_mutex. - */ - if (cgroup_is_populated(cgrp)) - return -EBUSY; - - /* - * Make sure there's no live children. We can't test emptiness of - * ->self.children as dead children linger on it while being - * drained; otherwise, "rmdir parent/child parent" may fail. - */ - if (css_has_online_children(&cgrp->self)) - return -EBUSY; - - /* - * Mark @cgrp and the associated csets dead. The former prevents - * further task migration and child creation by disabling - * cgroup_lock_live_group(). The latter makes the csets ignored by - * the migration path. - */ - cgrp->self.flags &= ~CSS_ONLINE; - - spin_lock_irq(&css_set_lock); - list_for_each_entry(link, &cgrp->cset_links, cset_link) - link->cset->dead = true; - spin_unlock_irq(&css_set_lock); - - /* initiate massacre of all css's */ - for_each_css(css, ssid, cgrp) - kill_css(css); - - /* - * Remove @cgrp directory along with the base files. @cgrp has an - * extra ref on its kn. - */ - kernfs_remove(cgrp->kn); - - check_for_release(cgroup_parent(cgrp)); - - /* put the base reference */ - percpu_ref_kill(&cgrp->self.refcnt); - - return 0; -}; - -static int cgroup_rmdir(struct kernfs_node *kn) -{ - struct cgroup *cgrp; - int ret = 0; - - cgrp = cgroup_kn_lock_live(kn, false); - if (!cgrp) - return 0; - - ret = cgroup_destroy_locked(cgrp); - - if (!ret) - trace_cgroup_rmdir(cgrp); - - cgroup_kn_unlock(kn); - return ret; -} - -static struct kernfs_syscall_ops cgroup_kf_syscall_ops = { - .remount_fs = cgroup_remount, - .show_options = cgroup_show_options, - .mkdir = cgroup_mkdir, - .rmdir = cgroup_rmdir, - .rename = cgroup_rename, - .show_path = cgroup_show_path, -}; - -static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early) -{ - struct cgroup_subsys_state *css; - - pr_debug("Initializing cgroup subsys %s\n", ss->name); - - mutex_lock(&cgroup_mutex); - - idr_init(&ss->css_idr); - INIT_LIST_HEAD(&ss->cfts); - - /* Create the root cgroup state for this subsystem */ - ss->root = &cgrp_dfl_root; - css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss)); - /* We don't handle early failures gracefully */ - BUG_ON(IS_ERR(css)); - init_and_link_css(css, ss, &cgrp_dfl_root.cgrp); - - /* - * Root csses are never destroyed and we can't initialize - * percpu_ref during early init. Disable refcnting. - */ - css->flags |= CSS_NO_REF; - - if (early) { - /* allocation can't be done safely during early init */ - css->id = 1; - } else { - css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL); - BUG_ON(css->id < 0); - } - - /* Update the init_css_set to contain a subsys - * pointer to this state - since the subsystem is - * newly registered, all tasks and hence the - * init_css_set is in the subsystem's root cgroup. */ - init_css_set.subsys[ss->id] = css; - - have_fork_callback |= (bool)ss->fork << ss->id; - have_exit_callback |= (bool)ss->exit << ss->id; - have_free_callback |= (bool)ss->free << ss->id; - have_canfork_callback |= (bool)ss->can_fork << ss->id; - - /* At system boot, before all subsystems have been - * registered, no tasks have been forked, so we don't - * need to invoke fork callbacks here. */ - BUG_ON(!list_empty(&init_task.tasks)); - - BUG_ON(online_css(css)); - - mutex_unlock(&cgroup_mutex); -} - -/** - * cgroup_init_early - cgroup initialization at system boot - * - * Initialize cgroups at system boot, and initialize any - * subsystems that request early init. - */ -int __init cgroup_init_early(void) -{ - static struct cgroup_sb_opts __initdata opts; - struct cgroup_subsys *ss; - int i; - - init_cgroup_root(&cgrp_dfl_root, &opts); - cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF; - - RCU_INIT_POINTER(init_task.cgroups, &init_css_set); - - for_each_subsys(ss, i) { - WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id, - "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p id:name=%d:%s\n", - i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free, - ss->id, ss->name); - WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN, - "cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]); - - ss->id = i; - ss->name = cgroup_subsys_name[i]; - if (!ss->legacy_name) - ss->legacy_name = cgroup_subsys_name[i]; - - if (ss->early_init) - cgroup_init_subsys(ss, true); - } - return 0; -} - -static u16 cgroup_disable_mask __initdata; - -/** - * cgroup_init - cgroup initialization - * - * Register cgroup filesystem and /proc file, and initialize - * any subsystems that didn't request early init. - */ -int __init cgroup_init(void) -{ - struct cgroup_subsys *ss; - int ssid; - - BUILD_BUG_ON(CGROUP_SUBSYS_COUNT > 16); - BUG_ON(percpu_init_rwsem(&cgroup_threadgroup_rwsem)); - BUG_ON(cgroup_init_cftypes(NULL, cgroup_dfl_base_files)); - BUG_ON(cgroup_init_cftypes(NULL, cgroup_legacy_base_files)); - - /* - * The latency of the synchronize_sched() is too high for cgroups, - * avoid it at the cost of forcing all readers into the slow path. - */ - rcu_sync_enter_start(&cgroup_threadgroup_rwsem.rss); - - get_user_ns(init_cgroup_ns.user_ns); - - mutex_lock(&cgroup_mutex); - - /* - * Add init_css_set to the hash table so that dfl_root can link to - * it during init. - */ - hash_add(css_set_table, &init_css_set.hlist, - css_set_hash(init_css_set.subsys)); - - BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0)); - - mutex_unlock(&cgroup_mutex); - - for_each_subsys(ss, ssid) { - if (ss->early_init) { - struct cgroup_subsys_state *css = - init_css_set.subsys[ss->id]; - - css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, - GFP_KERNEL); - BUG_ON(css->id < 0); - } else { - cgroup_init_subsys(ss, false); - } - - list_add_tail(&init_css_set.e_cset_node[ssid], - &cgrp_dfl_root.cgrp.e_csets[ssid]); - - /* - * Setting dfl_root subsys_mask needs to consider the - * disabled flag and cftype registration needs kmalloc, - * both of which aren't available during early_init. - */ - if (cgroup_disable_mask & (1 << ssid)) { - static_branch_disable(cgroup_subsys_enabled_key[ssid]); - printk(KERN_INFO "Disabling %s control group subsystem\n", - ss->name); - continue; - } - - if (cgroup_ssid_no_v1(ssid)) - printk(KERN_INFO "Disabling %s control group subsystem in v1 mounts\n", - ss->name); - - cgrp_dfl_root.subsys_mask |= 1 << ss->id; - - if (ss->implicit_on_dfl) - cgrp_dfl_implicit_ss_mask |= 1 << ss->id; - else if (!ss->dfl_cftypes) - cgrp_dfl_inhibit_ss_mask |= 1 << ss->id; - - if (ss->dfl_cftypes == ss->legacy_cftypes) { - WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes)); - } else { - WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes)); - WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes)); - } - - if (ss->bind) - ss->bind(init_css_set.subsys[ssid]); - } - - /* init_css_set.subsys[] has been updated, re-hash */ - hash_del(&init_css_set.hlist); - hash_add(css_set_table, &init_css_set.hlist, - css_set_hash(init_css_set.subsys)); - - WARN_ON(sysfs_create_mount_point(fs_kobj, "cgroup")); - WARN_ON(register_filesystem(&cgroup_fs_type)); - WARN_ON(register_filesystem(&cgroup2_fs_type)); - WARN_ON(!proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations)); - - return 0; -} - -static int __init cgroup_wq_init(void) -{ - /* - * There isn't much point in executing destruction path in - * parallel. Good chunk is serialized with cgroup_mutex anyway. - * Use 1 for @max_active. - * - * We would prefer to do this in cgroup_init() above, but that - * is called before init_workqueues(): so leave this until after. - */ - cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1); - BUG_ON(!cgroup_destroy_wq); - - /* - * Used to destroy pidlists and separate to serve as flush domain. - * Cap @max_active to 1 too. - */ - cgroup_pidlist_destroy_wq = alloc_workqueue("cgroup_pidlist_destroy", - 0, 1); - BUG_ON(!cgroup_pidlist_destroy_wq); - - return 0; -} -core_initcall(cgroup_wq_init); - -/* - * proc_cgroup_show() - * - Print task's cgroup paths into seq_file, one line for each hierarchy - * - Used for /proc//cgroup. - */ -int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns, - struct pid *pid, struct task_struct *tsk) -{ - char *buf; - int retval; - struct cgroup_root *root; - - retval = -ENOMEM; - buf = kmalloc(PATH_MAX, GFP_KERNEL); - if (!buf) - goto out; - - mutex_lock(&cgroup_mutex); - spin_lock_irq(&css_set_lock); - - for_each_root(root) { - struct cgroup_subsys *ss; - struct cgroup *cgrp; - int ssid, count = 0; - - if (root == &cgrp_dfl_root && !cgrp_dfl_visible) - continue; - - seq_printf(m, "%d:", root->hierarchy_id); - if (root != &cgrp_dfl_root) - for_each_subsys(ss, ssid) - if (root->subsys_mask & (1 << ssid)) - seq_printf(m, "%s%s", count++ ? "," : "", - ss->legacy_name); - if (strlen(root->name)) - seq_printf(m, "%sname=%s", count ? "," : "", - root->name); - seq_putc(m, ':'); - - cgrp = task_cgroup_from_root(tsk, root); - - /* - * On traditional hierarchies, all zombie tasks show up as - * belonging to the root cgroup. On the default hierarchy, - * while a zombie doesn't show up in "cgroup.procs" and - * thus can't be migrated, its /proc/PID/cgroup keeps - * reporting the cgroup it belonged to before exiting. If - * the cgroup is removed before the zombie is reaped, - * " (deleted)" is appended to the cgroup path. - */ - if (cgroup_on_dfl(cgrp) || !(tsk->flags & PF_EXITING)) { - retval = cgroup_path_ns_locked(cgrp, buf, PATH_MAX, - current->nsproxy->cgroup_ns); - if (retval >= PATH_MAX) - retval = -ENAMETOOLONG; - if (retval < 0) - goto out_unlock; - - seq_puts(m, buf); - } else { - seq_puts(m, "/"); - } - - if (cgroup_on_dfl(cgrp) && cgroup_is_dead(cgrp)) - seq_puts(m, " (deleted)\n"); - else - seq_putc(m, '\n'); - } - - retval = 0; -out_unlock: - spin_unlock_irq(&css_set_lock); - mutex_unlock(&cgroup_mutex); - kfree(buf); -out: - return retval; -} - -/* Display information about each subsystem and each hierarchy */ -static int proc_cgroupstats_show(struct seq_file *m, void *v) -{ - struct cgroup_subsys *ss; - int i; - - seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n"); - /* - * ideally we don't want subsystems moving around while we do this. - * cgroup_mutex is also necessary to guarantee an atomic snapshot of - * subsys/hierarchy state. - */ - mutex_lock(&cgroup_mutex); - - for_each_subsys(ss, i) - seq_printf(m, "%s\t%d\t%d\t%d\n", - ss->legacy_name, ss->root->hierarchy_id, - atomic_read(&ss->root->nr_cgrps), - cgroup_ssid_enabled(i)); - - mutex_unlock(&cgroup_mutex); - return 0; -} - -static int cgroupstats_open(struct inode *inode, struct file *file) -{ - return single_open(file, proc_cgroupstats_show, NULL); -} - -static const struct file_operations proc_cgroupstats_operations = { - .open = cgroupstats_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - -/** - * cgroup_fork - initialize cgroup related fields during copy_process() - * @child: pointer to task_struct of forking parent process. - * - * A task is associated with the init_css_set until cgroup_post_fork() - * attaches it to the parent's css_set. Empty cg_list indicates that - * @child isn't holding reference to its css_set. - */ -void cgroup_fork(struct task_struct *child) -{ - RCU_INIT_POINTER(child->cgroups, &init_css_set); - INIT_LIST_HEAD(&child->cg_list); -} - -/** - * cgroup_can_fork - called on a new task before the process is exposed - * @child: the task in question. - * - * This calls the subsystem can_fork() callbacks. If the can_fork() callback - * returns an error, the fork aborts with that error code. This allows for - * a cgroup subsystem to conditionally allow or deny new forks. - */ -int cgroup_can_fork(struct task_struct *child) -{ - struct cgroup_subsys *ss; - int i, j, ret; - - do_each_subsys_mask(ss, i, have_canfork_callback) { - ret = ss->can_fork(child); - if (ret) - goto out_revert; - } while_each_subsys_mask(); - - return 0; - -out_revert: - for_each_subsys(ss, j) { - if (j >= i) - break; - if (ss->cancel_fork) - ss->cancel_fork(child); - } - - return ret; -} - -/** - * cgroup_cancel_fork - called if a fork failed after cgroup_can_fork() - * @child: the task in question - * - * This calls the cancel_fork() callbacks if a fork failed *after* - * cgroup_can_fork() succeded. - */ -void cgroup_cancel_fork(struct task_struct *child) -{ - struct cgroup_subsys *ss; - int i; - - for_each_subsys(ss, i) - if (ss->cancel_fork) - ss->cancel_fork(child); -} - -/** - * cgroup_post_fork - called on a new task after adding it to the task list - * @child: the task in question - * - * Adds the task to the list running through its css_set if necessary and - * call the subsystem fork() callbacks. Has to be after the task is - * visible on the task list in case we race with the first call to - * cgroup_task_iter_start() - to guarantee that the new task ends up on its - * list. - */ -void cgroup_post_fork(struct task_struct *child) -{ - struct cgroup_subsys *ss; - int i; - - /* - * This may race against cgroup_enable_task_cg_lists(). As that - * function sets use_task_css_set_links before grabbing - * tasklist_lock and we just went through tasklist_lock to add - * @child, it's guaranteed that either we see the set - * use_task_css_set_links or cgroup_enable_task_cg_lists() sees - * @child during its iteration. - * - * If we won the race, @child is associated with %current's - * css_set. Grabbing css_set_lock guarantees both that the - * association is stable, and, on completion of the parent's - * migration, @child is visible in the source of migration or - * already in the destination cgroup. This guarantee is necessary - * when implementing operations which need to migrate all tasks of - * a cgroup to another. - * - * Note that if we lose to cgroup_enable_task_cg_lists(), @child - * will remain in init_css_set. This is safe because all tasks are - * in the init_css_set before cg_links is enabled and there's no - * operation which transfers all tasks out of init_css_set. - */ - if (use_task_css_set_links) { - struct css_set *cset; - - spin_lock_irq(&css_set_lock); - cset = task_css_set(current); - if (list_empty(&child->cg_list)) { - get_css_set(cset); - css_set_move_task(child, NULL, cset, false); - } - spin_unlock_irq(&css_set_lock); - } - - /* - * Call ss->fork(). This must happen after @child is linked on - * css_set; otherwise, @child might change state between ->fork() - * and addition to css_set. - */ - do_each_subsys_mask(ss, i, have_fork_callback) { - ss->fork(child); - } while_each_subsys_mask(); -} - -/** - * cgroup_exit - detach cgroup from exiting task - * @tsk: pointer to task_struct of exiting process - * - * Description: Detach cgroup from @tsk and release it. - * - * Note that cgroups marked notify_on_release force every task in - * them to take the global cgroup_mutex mutex when exiting. - * This could impact scaling on very large systems. Be reluctant to - * use notify_on_release cgroups where very high task exit scaling - * is required on large systems. - * - * We set the exiting tasks cgroup to the root cgroup (top_cgroup). We - * call cgroup_exit() while the task is still competent to handle - * notify_on_release(), then leave the task attached to the root cgroup in - * each hierarchy for the remainder of its exit. No need to bother with - * init_css_set refcnting. init_css_set never goes away and we can't race - * with migration path - PF_EXITING is visible to migration path. - */ -void cgroup_exit(struct task_struct *tsk) -{ - struct cgroup_subsys *ss; - struct css_set *cset; - int i; - - /* - * Unlink from @tsk from its css_set. As migration path can't race - * with us, we can check css_set and cg_list without synchronization. - */ - cset = task_css_set(tsk); - - if (!list_empty(&tsk->cg_list)) { - spin_lock_irq(&css_set_lock); - css_set_move_task(tsk, cset, NULL, false); - spin_unlock_irq(&css_set_lock); - } else { - get_css_set(cset); - } - - /* see cgroup_post_fork() for details */ - do_each_subsys_mask(ss, i, have_exit_callback) { - ss->exit(tsk); - } while_each_subsys_mask(); -} - -void cgroup_free(struct task_struct *task) -{ - struct css_set *cset = task_css_set(task); - struct cgroup_subsys *ss; - int ssid; - - do_each_subsys_mask(ss, ssid, have_free_callback) { - ss->free(task); - } while_each_subsys_mask(); - - put_css_set(cset); -} - -static void check_for_release(struct cgroup *cgrp) -{ - if (notify_on_release(cgrp) && !cgroup_is_populated(cgrp) && - !css_has_online_children(&cgrp->self) && !cgroup_is_dead(cgrp)) - schedule_work(&cgrp->release_agent_work); -} - -/* - * Notify userspace when a cgroup is released, by running the - * configured release agent with the name of the cgroup (path - * relative to the root of cgroup file system) as the argument. - * - * Most likely, this user command will try to rmdir this cgroup. - * - * This races with the possibility that some other task will be - * attached to this cgroup before it is removed, or that some other - * user task will 'mkdir' a child cgroup of this cgroup. That's ok. - * The presumed 'rmdir' will fail quietly if this cgroup is no longer - * unused, and this cgroup will be reprieved from its death sentence, - * to continue to serve a useful existence. Next time it's released, - * we will get notified again, if it still has 'notify_on_release' set. - * - * The final arg to call_usermodehelper() is UMH_WAIT_EXEC, which - * means only wait until the task is successfully execve()'d. The - * separate release agent task is forked by call_usermodehelper(), - * then control in this thread returns here, without waiting for the - * release agent task. We don't bother to wait because the caller of - * this routine has no use for the exit status of the release agent - * task, so no sense holding our caller up for that. - */ -static void cgroup_release_agent(struct work_struct *work) -{ - struct cgroup *cgrp = - container_of(work, struct cgroup, release_agent_work); - char *pathbuf = NULL, *agentbuf = NULL; - char *argv[3], *envp[3]; - int ret; - - mutex_lock(&cgroup_mutex); - - pathbuf = kmalloc(PATH_MAX, GFP_KERNEL); - agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL); - if (!pathbuf || !agentbuf) - goto out; - - spin_lock_irq(&css_set_lock); - ret = cgroup_path_ns_locked(cgrp, pathbuf, PATH_MAX, &init_cgroup_ns); - spin_unlock_irq(&css_set_lock); - if (ret < 0 || ret >= PATH_MAX) - goto out; - - argv[0] = agentbuf; - argv[1] = pathbuf; - argv[2] = NULL; - - /* minimal command environment */ - envp[0] = "HOME=/"; - envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin"; - envp[2] = NULL; - - mutex_unlock(&cgroup_mutex); - call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC); - goto out_free; -out: - mutex_unlock(&cgroup_mutex); -out_free: - kfree(agentbuf); - kfree(pathbuf); -} - -static int __init cgroup_disable(char *str) -{ - struct cgroup_subsys *ss; - char *token; - int i; - - while ((token = strsep(&str, ",")) != NULL) { - if (!*token) - continue; - - for_each_subsys(ss, i) { - if (strcmp(token, ss->name) && - strcmp(token, ss->legacy_name)) - continue; - cgroup_disable_mask |= 1 << i; - } - } - return 1; -} -__setup("cgroup_disable=", cgroup_disable); - -static int __init cgroup_no_v1(char *str) -{ - struct cgroup_subsys *ss; - char *token; - int i; - - while ((token = strsep(&str, ",")) != NULL) { - if (!*token) - continue; - - if (!strcmp(token, "all")) { - cgroup_no_v1_mask = U16_MAX; - break; - } - - for_each_subsys(ss, i) { - if (strcmp(token, ss->name) && - strcmp(token, ss->legacy_name)) - continue; - - cgroup_no_v1_mask |= 1 << i; - } - } - return 1; -} -__setup("cgroup_no_v1=", cgroup_no_v1); - -/** - * css_tryget_online_from_dir - get corresponding css from a cgroup dentry - * @dentry: directory dentry of interest - * @ss: subsystem of interest - * - * If @dentry is a directory for a cgroup which has @ss enabled on it, try - * to get the corresponding css and return it. If such css doesn't exist - * or can't be pinned, an ERR_PTR value is returned. - */ -struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry, - struct cgroup_subsys *ss) -{ - struct kernfs_node *kn = kernfs_node_from_dentry(dentry); - struct file_system_type *s_type = dentry->d_sb->s_type; - struct cgroup_subsys_state *css = NULL; - struct cgroup *cgrp; - - /* is @dentry a cgroup dir? */ - if ((s_type != &cgroup_fs_type && s_type != &cgroup2_fs_type) || - !kn || kernfs_type(kn) != KERNFS_DIR) - return ERR_PTR(-EBADF); - - rcu_read_lock(); - - /* - * This path doesn't originate from kernfs and @kn could already - * have been or be removed at any point. @kn->priv is RCU - * protected for this access. See css_release_work_fn() for details. - */ - cgrp = rcu_dereference(kn->priv); - if (cgrp) - css = cgroup_css(cgrp, ss); - - if (!css || !css_tryget_online(css)) - css = ERR_PTR(-ENOENT); - - rcu_read_unlock(); - return css; -} - -/** - * css_from_id - lookup css by id - * @id: the cgroup id - * @ss: cgroup subsys to be looked into - * - * Returns the css if there's valid one with @id, otherwise returns NULL. - * Should be called under rcu_read_lock(). - */ -struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss) -{ - WARN_ON_ONCE(!rcu_read_lock_held()); - return idr_find(&ss->css_idr, id); -} - -/** - * cgroup_get_from_path - lookup and get a cgroup from its default hierarchy path - * @path: path on the default hierarchy - * - * Find the cgroup at @path on the default hierarchy, increment its - * reference count and return it. Returns pointer to the found cgroup on - * success, ERR_PTR(-ENOENT) if @path doens't exist and ERR_PTR(-ENOTDIR) - * if @path points to a non-directory. - */ -struct cgroup *cgroup_get_from_path(const char *path) -{ - struct kernfs_node *kn; - struct cgroup *cgrp; - - mutex_lock(&cgroup_mutex); - - kn = kernfs_walk_and_get(cgrp_dfl_root.cgrp.kn, path); - if (kn) { - if (kernfs_type(kn) == KERNFS_DIR) { - cgrp = kn->priv; - cgroup_get(cgrp); - } else { - cgrp = ERR_PTR(-ENOTDIR); - } - kernfs_put(kn); - } else { - cgrp = ERR_PTR(-ENOENT); - } - - mutex_unlock(&cgroup_mutex); - return cgrp; -} -EXPORT_SYMBOL_GPL(cgroup_get_from_path); - -/** - * cgroup_get_from_fd - get a cgroup pointer from a fd - * @fd: fd obtained by open(cgroup2_dir) - * - * Find the cgroup from a fd which should be obtained - * by opening a cgroup directory. Returns a pointer to the - * cgroup on success. ERR_PTR is returned if the cgroup - * cannot be found. - */ -struct cgroup *cgroup_get_from_fd(int fd) -{ - struct cgroup_subsys_state *css; - struct cgroup *cgrp; - struct file *f; - - f = fget_raw(fd); - if (!f) - return ERR_PTR(-EBADF); - - css = css_tryget_online_from_dir(f->f_path.dentry, NULL); - fput(f); - if (IS_ERR(css)) - return ERR_CAST(css); - - cgrp = css->cgroup; - if (!cgroup_on_dfl(cgrp)) { - cgroup_put(cgrp); - return ERR_PTR(-EBADF); - } - - return cgrp; -} -EXPORT_SYMBOL_GPL(cgroup_get_from_fd); - -/* - * sock->sk_cgrp_data handling. For more info, see sock_cgroup_data - * definition in cgroup-defs.h. - */ -#ifdef CONFIG_SOCK_CGROUP_DATA - -#if defined(CONFIG_CGROUP_NET_PRIO) || defined(CONFIG_CGROUP_NET_CLASSID) - -DEFINE_SPINLOCK(cgroup_sk_update_lock); -static bool cgroup_sk_alloc_disabled __read_mostly; - -void cgroup_sk_alloc_disable(void) -{ - if (cgroup_sk_alloc_disabled) - return; - pr_info("cgroup: disabling cgroup2 socket matching due to net_prio or net_cls activation\n"); - cgroup_sk_alloc_disabled = true; -} - -#else - -#define cgroup_sk_alloc_disabled false - -#endif - -void cgroup_sk_alloc(struct sock_cgroup_data *skcd) -{ - if (cgroup_sk_alloc_disabled) - return; - - /* Socket clone path */ - if (skcd->val) { - cgroup_get(sock_cgroup_ptr(skcd)); - return; - } - - rcu_read_lock(); - - while (true) { - struct css_set *cset; - - cset = task_css_set(current); - if (likely(cgroup_tryget(cset->dfl_cgrp))) { - skcd->val = (unsigned long)cset->dfl_cgrp; - break; - } - cpu_relax(); - } - - rcu_read_unlock(); -} - -void cgroup_sk_free(struct sock_cgroup_data *skcd) -{ - cgroup_put(sock_cgroup_ptr(skcd)); -} - -#endif /* CONFIG_SOCK_CGROUP_DATA */ - -/* cgroup namespaces */ - -static struct ucounts *inc_cgroup_namespaces(struct user_namespace *ns) -{ - return inc_ucount(ns, current_euid(), UCOUNT_CGROUP_NAMESPACES); -} - -static void dec_cgroup_namespaces(struct ucounts *ucounts) -{ - dec_ucount(ucounts, UCOUNT_CGROUP_NAMESPACES); -} - -static struct cgroup_namespace *alloc_cgroup_ns(void) -{ - struct cgroup_namespace *new_ns; - int ret; - - new_ns = kzalloc(sizeof(struct cgroup_namespace), GFP_KERNEL); - if (!new_ns) - return ERR_PTR(-ENOMEM); - ret = ns_alloc_inum(&new_ns->ns); - if (ret) { - kfree(new_ns); - return ERR_PTR(ret); - } - atomic_set(&new_ns->count, 1); - new_ns->ns.ops = &cgroupns_operations; - return new_ns; -} - -void free_cgroup_ns(struct cgroup_namespace *ns) -{ - put_css_set(ns->root_cset); - dec_cgroup_namespaces(ns->ucounts); - put_user_ns(ns->user_ns); - ns_free_inum(&ns->ns); - kfree(ns); -} -EXPORT_SYMBOL(free_cgroup_ns); - -struct cgroup_namespace *copy_cgroup_ns(unsigned long flags, - struct user_namespace *user_ns, - struct cgroup_namespace *old_ns) -{ - struct cgroup_namespace *new_ns; - struct ucounts *ucounts; - struct css_set *cset; - - BUG_ON(!old_ns); - - if (!(flags & CLONE_NEWCGROUP)) { - get_cgroup_ns(old_ns); - return old_ns; - } - - /* Allow only sysadmin to create cgroup namespace. */ - if (!ns_capable(user_ns, CAP_SYS_ADMIN)) - return ERR_PTR(-EPERM); - - ucounts = inc_cgroup_namespaces(user_ns); - if (!ucounts) - return ERR_PTR(-ENOSPC); - - /* It is not safe to take cgroup_mutex here */ - spin_lock_irq(&css_set_lock); - cset = task_css_set(current); - get_css_set(cset); - spin_unlock_irq(&css_set_lock); - - new_ns = alloc_cgroup_ns(); - if (IS_ERR(new_ns)) { - put_css_set(cset); - dec_cgroup_namespaces(ucounts); - return new_ns; - } - - new_ns->user_ns = get_user_ns(user_ns); - new_ns->ucounts = ucounts; - new_ns->root_cset = cset; - - return new_ns; -} - -static inline struct cgroup_namespace *to_cg_ns(struct ns_common *ns) -{ - return container_of(ns, struct cgroup_namespace, ns); -} - -static int cgroupns_install(struct nsproxy *nsproxy, struct ns_common *ns) -{ - struct cgroup_namespace *cgroup_ns = to_cg_ns(ns); - - if (!ns_capable(current_user_ns(), CAP_SYS_ADMIN) || - !ns_capable(cgroup_ns->user_ns, CAP_SYS_ADMIN)) - return -EPERM; - - /* Don't need to do anything if we are attaching to our own cgroupns. */ - if (cgroup_ns == nsproxy->cgroup_ns) - return 0; - - get_cgroup_ns(cgroup_ns); - put_cgroup_ns(nsproxy->cgroup_ns); - nsproxy->cgroup_ns = cgroup_ns; - - return 0; -} - -static struct ns_common *cgroupns_get(struct task_struct *task) -{ - struct cgroup_namespace *ns = NULL; - struct nsproxy *nsproxy; - - task_lock(task); - nsproxy = task->nsproxy; - if (nsproxy) { - ns = nsproxy->cgroup_ns; - get_cgroup_ns(ns); - } - task_unlock(task); - - return ns ? &ns->ns : NULL; -} - -static void cgroupns_put(struct ns_common *ns) -{ - put_cgroup_ns(to_cg_ns(ns)); -} - -static struct user_namespace *cgroupns_owner(struct ns_common *ns) -{ - return to_cg_ns(ns)->user_ns; -} - -const struct proc_ns_operations cgroupns_operations = { - .name = "cgroup", - .type = CLONE_NEWCGROUP, - .get = cgroupns_get, - .put = cgroupns_put, - .install = cgroupns_install, - .owner = cgroupns_owner, -}; - -static __init int cgroup_namespaces_init(void) -{ - return 0; -} -subsys_initcall(cgroup_namespaces_init); - -#ifdef CONFIG_CGROUP_BPF -void cgroup_bpf_update(struct cgroup *cgrp, - struct bpf_prog *prog, - enum bpf_attach_type type) -{ - struct cgroup *parent = cgroup_parent(cgrp); - - mutex_lock(&cgroup_mutex); - __cgroup_bpf_update(cgrp, parent, prog, type); - mutex_unlock(&cgroup_mutex); -} -#endif /* CONFIG_CGROUP_BPF */ - -#ifdef CONFIG_CGROUP_DEBUG -static struct cgroup_subsys_state * -debug_css_alloc(struct cgroup_subsys_state *parent_css) -{ - struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL); - - if (!css) - return ERR_PTR(-ENOMEM); - - return css; -} - -static void debug_css_free(struct cgroup_subsys_state *css) -{ - kfree(css); -} - -static u64 debug_taskcount_read(struct cgroup_subsys_state *css, - struct cftype *cft) -{ - return cgroup_task_count(css->cgroup); -} - -static u64 current_css_set_read(struct cgroup_subsys_state *css, - struct cftype *cft) -{ - return (u64)(unsigned long)current->cgroups; -} - -static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css, - struct cftype *cft) -{ - u64 count; - - rcu_read_lock(); - count = atomic_read(&task_css_set(current)->refcount); - rcu_read_unlock(); - return count; -} - -static int current_css_set_cg_links_read(struct seq_file *seq, void *v) -{ - struct cgrp_cset_link *link; - struct css_set *cset; - char *name_buf; - - name_buf = kmalloc(NAME_MAX + 1, GFP_KERNEL); - if (!name_buf) - return -ENOMEM; - - spin_lock_irq(&css_set_lock); - rcu_read_lock(); - cset = rcu_dereference(current->cgroups); - list_for_each_entry(link, &cset->cgrp_links, cgrp_link) { - struct cgroup *c = link->cgrp; - - cgroup_name(c, name_buf, NAME_MAX + 1); - seq_printf(seq, "Root %d group %s\n", - c->root->hierarchy_id, name_buf); - } - rcu_read_unlock(); - spin_unlock_irq(&css_set_lock); - kfree(name_buf); - return 0; -} - -#define MAX_TASKS_SHOWN_PER_CSS 25 -static int cgroup_css_links_read(struct seq_file *seq, void *v) -{ - struct cgroup_subsys_state *css = seq_css(seq); - struct cgrp_cset_link *link; - - spin_lock_irq(&css_set_lock); - list_for_each_entry(link, &css->cgroup->cset_links, cset_link) { - struct css_set *cset = link->cset; - struct task_struct *task; - int count = 0; - - seq_printf(seq, "css_set %p\n", cset); - - list_for_each_entry(task, &cset->tasks, cg_list) { - if (count++ > MAX_TASKS_SHOWN_PER_CSS) - goto overflow; - seq_printf(seq, " task %d\n", task_pid_vnr(task)); - } - - list_for_each_entry(task, &cset->mg_tasks, cg_list) { - if (count++ > MAX_TASKS_SHOWN_PER_CSS) - goto overflow; - seq_printf(seq, " task %d\n", task_pid_vnr(task)); - } - continue; - overflow: - seq_puts(seq, " ...\n"); - } - spin_unlock_irq(&css_set_lock); - return 0; -} - -static u64 releasable_read(struct cgroup_subsys_state *css, struct cftype *cft) -{ - return (!cgroup_is_populated(css->cgroup) && - !css_has_online_children(&css->cgroup->self)); -} - -static struct cftype debug_files[] = { - { - .name = "taskcount", - .read_u64 = debug_taskcount_read, - }, - - { - .name = "current_css_set", - .read_u64 = current_css_set_read, - }, - - { - .name = "current_css_set_refcount", - .read_u64 = current_css_set_refcount_read, - }, - - { - .name = "current_css_set_cg_links", - .seq_show = current_css_set_cg_links_read, - }, - - { - .name = "cgroup_css_links", - .seq_show = cgroup_css_links_read, - }, - - { - .name = "releasable", - .read_u64 = releasable_read, - }, - - { } /* terminate */ -}; - -struct cgroup_subsys debug_cgrp_subsys = { - .css_alloc = debug_css_alloc, - .css_free = debug_css_free, - .legacy_cftypes = debug_files, -}; -#endif /* CONFIG_CGROUP_DEBUG */ diff --git a/kernel/cgroup/Makefile b/kernel/cgroup/Makefile new file mode 100644 index 000000000000..4d561a50a5ac --- /dev/null +++ b/kernel/cgroup/Makefile @@ -0,0 +1,5 @@ +obj-y := cgroup.o + +obj-$(CONFIG_CGROUP_FREEZER) += freezer.o +obj-$(CONFIG_CGROUP_PIDS) += pids.o +obj-$(CONFIG_CPUSETS) += cpuset.o diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c new file mode 100644 index 000000000000..1a815f275849 --- /dev/null +++ b/kernel/cgroup/cgroup.c @@ -0,0 +1,6705 @@ +/* + * Generic process-grouping system. + * + * Based originally on the cpuset system, extracted by Paul Menage + * Copyright (C) 2006 Google, Inc + * + * Notifications support + * Copyright (C) 2009 Nokia Corporation + * Author: Kirill A. Shutemov + * + * Copyright notices from the original cpuset code: + * -------------------------------------------------- + * Copyright (C) 2003 BULL SA. + * Copyright (C) 2004-2006 Silicon Graphics, Inc. + * + * Portions derived from Patrick Mochel's sysfs code. + * sysfs is Copyright (c) 2001-3 Patrick Mochel + * + * 2003-10-10 Written by Simon Derr. + * 2003-10-22 Updates by Stephen Hemminger. + * 2004 May-July Rework by Paul Jackson. + * --------------------------------------------------- + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file COPYING in the main directory of the Linux + * distribution for more details. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* TODO: replace with more sophisticated array */ +#include +#include +#include +#include +#include +#include +#include +#include + +#define CREATE_TRACE_POINTS +#include + +/* + * pidlists linger the following amount before being destroyed. The goal + * is avoiding frequent destruction in the middle of consecutive read calls + * Expiring in the middle is a performance problem not a correctness one. + * 1 sec should be enough. + */ +#define CGROUP_PIDLIST_DESTROY_DELAY HZ + +#define CGROUP_FILE_NAME_MAX (MAX_CGROUP_TYPE_NAMELEN + \ + MAX_CFTYPE_NAME + 2) + +/* + * cgroup_mutex is the master lock. Any modification to cgroup or its + * hierarchy must be performed while holding it. + * + * css_set_lock protects task->cgroups pointer, the list of css_set + * objects, and the chain of tasks off each css_set. + * + * These locks are exported if CONFIG_PROVE_RCU so that accessors in + * cgroup.h can use them for lockdep annotations. + */ +#ifdef CONFIG_PROVE_RCU +DEFINE_MUTEX(cgroup_mutex); +DEFINE_SPINLOCK(css_set_lock); +EXPORT_SYMBOL_GPL(cgroup_mutex); +EXPORT_SYMBOL_GPL(css_set_lock); +#else +static DEFINE_MUTEX(cgroup_mutex); +static DEFINE_SPINLOCK(css_set_lock); +#endif + +/* + * Protects cgroup_idr and css_idr so that IDs can be released without + * grabbing cgroup_mutex. + */ +static DEFINE_SPINLOCK(cgroup_idr_lock); + +/* + * Protects cgroup_file->kn for !self csses. It synchronizes notifications + * against file removal/re-creation across css hiding. + */ +static DEFINE_SPINLOCK(cgroup_file_kn_lock); + +/* + * Protects cgroup_subsys->release_agent_path. Modifying it also requires + * cgroup_mutex. Reading requires either cgroup_mutex or this spinlock. + */ +static DEFINE_SPINLOCK(release_agent_path_lock); + +struct percpu_rw_semaphore cgroup_threadgroup_rwsem; + +#define cgroup_assert_mutex_or_rcu_locked() \ + RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \ + !lockdep_is_held(&cgroup_mutex), \ + "cgroup_mutex or RCU read lock required"); + +/* + * cgroup destruction makes heavy use of work items and there can be a lot + * of concurrent destructions. Use a separate workqueue so that cgroup + * destruction work items don't end up filling up max_active of system_wq + * which may lead to deadlock. + */ +static struct workqueue_struct *cgroup_destroy_wq; + +/* + * pidlist destructions need to be flushed on cgroup destruction. Use a + * separate workqueue as flush domain. + */ +static struct workqueue_struct *cgroup_pidlist_destroy_wq; + +/* generate an array of cgroup subsystem pointers */ +#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys, +static struct cgroup_subsys *cgroup_subsys[] = { +#include +}; +#undef SUBSYS + +/* array of cgroup subsystem names */ +#define SUBSYS(_x) [_x ## _cgrp_id] = #_x, +static const char *cgroup_subsys_name[] = { +#include +}; +#undef SUBSYS + +/* array of static_keys for cgroup_subsys_enabled() and cgroup_subsys_on_dfl() */ +#define SUBSYS(_x) \ + DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_enabled_key); \ + DEFINE_STATIC_KEY_TRUE(_x ## _cgrp_subsys_on_dfl_key); \ + EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_enabled_key); \ + EXPORT_SYMBOL_GPL(_x ## _cgrp_subsys_on_dfl_key); +#include +#undef SUBSYS + +#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_enabled_key, +static struct static_key_true *cgroup_subsys_enabled_key[] = { +#include +}; +#undef SUBSYS + +#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys_on_dfl_key, +static struct static_key_true *cgroup_subsys_on_dfl_key[] = { +#include +}; +#undef SUBSYS + +/* + * The default hierarchy, reserved for the subsystems that are otherwise + * unattached - it never has more than a single cgroup, and all tasks are + * part of that cgroup. + */ +struct cgroup_root cgrp_dfl_root; +EXPORT_SYMBOL_GPL(cgrp_dfl_root); + +/* + * The default hierarchy always exists but is hidden until mounted for the + * first time. This is for backward compatibility. + */ +static bool cgrp_dfl_visible; + +/* Controllers blocked by the commandline in v1 */ +static u16 cgroup_no_v1_mask; + +/* some controllers are not supported in the default hierarchy */ +static u16 cgrp_dfl_inhibit_ss_mask; + +/* some controllers are implicitly enabled on the default hierarchy */ +static unsigned long cgrp_dfl_implicit_ss_mask; + +/* The list of hierarchy roots */ + +static LIST_HEAD(cgroup_roots); +static int cgroup_root_count; + +/* hierarchy ID allocation and mapping, protected by cgroup_mutex */ +static DEFINE_IDR(cgroup_hierarchy_idr); + +/* + * Assign a monotonically increasing serial number to csses. It guarantees + * cgroups with bigger numbers are newer than those with smaller numbers. + * Also, as csses are always appended to the parent's ->children list, it + * guarantees that sibling csses are always sorted in the ascending serial + * number order on the list. Protected by cgroup_mutex. + */ +static u64 css_serial_nr_next = 1; + +/* + * These bitmask flags indicate whether tasks in the fork and exit paths have + * fork/exit handlers to call. This avoids us having to do extra work in the + * fork/exit path to check which subsystems have fork/exit callbacks. + */ +static u16 have_fork_callback __read_mostly; +static u16 have_exit_callback __read_mostly; +static u16 have_free_callback __read_mostly; + +/* cgroup namespace for init task */ +struct cgroup_namespace init_cgroup_ns = { + .count = { .counter = 2, }, + .user_ns = &init_user_ns, + .ns.ops = &cgroupns_operations, + .ns.inum = PROC_CGROUP_INIT_INO, + .root_cset = &init_css_set, +}; + +/* Ditto for the can_fork callback. */ +static u16 have_canfork_callback __read_mostly; + +static struct file_system_type cgroup2_fs_type; +static struct cftype cgroup_dfl_base_files[]; +static struct cftype cgroup_legacy_base_files[]; + +static int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask); +static void cgroup_lock_and_drain_offline(struct cgroup *cgrp); +static int cgroup_apply_control(struct cgroup *cgrp); +static void cgroup_finalize_control(struct cgroup *cgrp, int ret); +static void css_task_iter_advance(struct css_task_iter *it); +static int cgroup_destroy_locked(struct cgroup *cgrp); +static struct cgroup_subsys_state *css_create(struct cgroup *cgrp, + struct cgroup_subsys *ss); +static void css_release(struct percpu_ref *ref); +static void kill_css(struct cgroup_subsys_state *css); +static int cgroup_addrm_files(struct cgroup_subsys_state *css, + struct cgroup *cgrp, struct cftype cfts[], + bool is_add); + +/** + * cgroup_ssid_enabled - cgroup subsys enabled test by subsys ID + * @ssid: subsys ID of interest + * + * cgroup_subsys_enabled() can only be used with literal subsys names which + * is fine for individual subsystems but unsuitable for cgroup core. This + * is slower static_key_enabled() based test indexed by @ssid. + */ +static bool cgroup_ssid_enabled(int ssid) +{ + if (CGROUP_SUBSYS_COUNT == 0) + return false; + + return static_key_enabled(cgroup_subsys_enabled_key[ssid]); +} + +static bool cgroup_ssid_no_v1(int ssid) +{ + return cgroup_no_v1_mask & (1 << ssid); +} + +/** + * cgroup_on_dfl - test whether a cgroup is on the default hierarchy + * @cgrp: the cgroup of interest + * + * The default hierarchy is the v2 interface of cgroup and this function + * can be used to test whether a cgroup is on the default hierarchy for + * cases where a subsystem should behave differnetly depending on the + * interface version. + * + * The set of behaviors which change on the default hierarchy are still + * being determined and the mount option is prefixed with __DEVEL__. + * + * List of changed behaviors: + * + * - Mount options "noprefix", "xattr", "clone_children", "release_agent" + * and "name" are disallowed. + * + * - When mounting an existing superblock, mount options should match. + * + * - Remount is disallowed. + * + * - rename(2) is disallowed. + * + * - "tasks" is removed. Everything should be at process granularity. Use + * "cgroup.procs" instead. + * + * - "cgroup.procs" is not sorted. pids will be unique unless they got + * recycled inbetween reads. + * + * - "release_agent" and "notify_on_release" are removed. Replacement + * notification mechanism will be implemented. + * + * - "cgroup.clone_children" is removed. + * + * - "cgroup.subtree_populated" is available. Its value is 0 if the cgroup + * and its descendants contain no task; otherwise, 1. The file also + * generates kernfs notification which can be monitored through poll and + * [di]notify when the value of the file changes. + * + * - cpuset: tasks will be kept in empty cpusets when hotplug happens and + * take masks of ancestors with non-empty cpus/mems, instead of being + * moved to an ancestor. + * + * - cpuset: a task can be moved into an empty cpuset, and again it takes + * masks of ancestors. + * + * - memcg: use_hierarchy is on by default and the cgroup file for the flag + * is not created. + * + * - blkcg: blk-throttle becomes properly hierarchical. + * + * - debug: disallowed on the default hierarchy. + */ +static bool cgroup_on_dfl(const struct cgroup *cgrp) +{ + return cgrp->root == &cgrp_dfl_root; +} + +/* IDR wrappers which synchronize using cgroup_idr_lock */ +static int cgroup_idr_alloc(struct idr *idr, void *ptr, int start, int end, + gfp_t gfp_mask) +{ + int ret; + + idr_preload(gfp_mask); + spin_lock_bh(&cgroup_idr_lock); + ret = idr_alloc(idr, ptr, start, end, gfp_mask & ~__GFP_DIRECT_RECLAIM); + spin_unlock_bh(&cgroup_idr_lock); + idr_preload_end(); + return ret; +} + +static void *cgroup_idr_replace(struct idr *idr, void *ptr, int id) +{ + void *ret; + + spin_lock_bh(&cgroup_idr_lock); + ret = idr_replace(idr, ptr, id); + spin_unlock_bh(&cgroup_idr_lock); + return ret; +} + +static void cgroup_idr_remove(struct idr *idr, int id) +{ + spin_lock_bh(&cgroup_idr_lock); + idr_remove(idr, id); + spin_unlock_bh(&cgroup_idr_lock); +} + +static struct cgroup *cgroup_parent(struct cgroup *cgrp) +{ + struct cgroup_subsys_state *parent_css = cgrp->self.parent; + + if (parent_css) + return container_of(parent_css, struct cgroup, self); + return NULL; +} + +/* subsystems visibly enabled on a cgroup */ +static u16 cgroup_control(struct cgroup *cgrp) +{ + struct cgroup *parent = cgroup_parent(cgrp); + u16 root_ss_mask = cgrp->root->subsys_mask; + + if (parent) + return parent->subtree_control; + + if (cgroup_on_dfl(cgrp)) + root_ss_mask &= ~(cgrp_dfl_inhibit_ss_mask | + cgrp_dfl_implicit_ss_mask); + return root_ss_mask; +} + +/* subsystems enabled on a cgroup */ +static u16 cgroup_ss_mask(struct cgroup *cgrp) +{ + struct cgroup *parent = cgroup_parent(cgrp); + + if (parent) + return parent->subtree_ss_mask; + + return cgrp->root->subsys_mask; +} + +/** + * cgroup_css - obtain a cgroup's css for the specified subsystem + * @cgrp: the cgroup of interest + * @ss: the subsystem of interest (%NULL returns @cgrp->self) + * + * Return @cgrp's css (cgroup_subsys_state) associated with @ss. This + * function must be called either under cgroup_mutex or rcu_read_lock() and + * the caller is responsible for pinning the returned css if it wants to + * keep accessing it outside the said locks. This function may return + * %NULL if @cgrp doesn't have @subsys_id enabled. + */ +static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp, + struct cgroup_subsys *ss) +{ + if (ss) + return rcu_dereference_check(cgrp->subsys[ss->id], + lockdep_is_held(&cgroup_mutex)); + else + return &cgrp->self; +} + +/** + * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem + * @cgrp: the cgroup of interest + * @ss: the subsystem of interest (%NULL returns @cgrp->self) + * + * Similar to cgroup_css() but returns the effective css, which is defined + * as the matching css of the nearest ancestor including self which has @ss + * enabled. If @ss is associated with the hierarchy @cgrp is on, this + * function is guaranteed to return non-NULL css. + */ +static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp, + struct cgroup_subsys *ss) +{ + lockdep_assert_held(&cgroup_mutex); + + if (!ss) + return &cgrp->self; + + /* + * This function is used while updating css associations and thus + * can't test the csses directly. Test ss_mask. + */ + while (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) { + cgrp = cgroup_parent(cgrp); + if (!cgrp) + return NULL; + } + + return cgroup_css(cgrp, ss); +} + +/** + * cgroup_get_e_css - get a cgroup's effective css for the specified subsystem + * @cgrp: the cgroup of interest + * @ss: the subsystem of interest + * + * Find and get the effective css of @cgrp for @ss. The effective css is + * defined as the matching css of the nearest ancestor including self which + * has @ss enabled. If @ss is not mounted on the hierarchy @cgrp is on, + * the root css is returned, so this function always returns a valid css. + * The returned css must be put using css_put(). + */ +struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgrp, + struct cgroup_subsys *ss) +{ + struct cgroup_subsys_state *css; + + rcu_read_lock(); + + do { + css = cgroup_css(cgrp, ss); + + if (css && css_tryget_online(css)) + goto out_unlock; + cgrp = cgroup_parent(cgrp); + } while (cgrp); + + css = init_css_set.subsys[ss->id]; + css_get(css); +out_unlock: + rcu_read_unlock(); + return css; +} + +/* convenient tests for these bits */ +static inline bool cgroup_is_dead(const struct cgroup *cgrp) +{ + return !(cgrp->self.flags & CSS_ONLINE); +} + +static void cgroup_get(struct cgroup *cgrp) +{ + WARN_ON_ONCE(cgroup_is_dead(cgrp)); + css_get(&cgrp->self); +} + +static bool cgroup_tryget(struct cgroup *cgrp) +{ + return css_tryget(&cgrp->self); +} + +struct cgroup_subsys_state *of_css(struct kernfs_open_file *of) +{ + struct cgroup *cgrp = of->kn->parent->priv; + struct cftype *cft = of_cft(of); + + /* + * This is open and unprotected implementation of cgroup_css(). + * seq_css() is only called from a kernfs file operation which has + * an active reference on the file. Because all the subsystem + * files are drained before a css is disassociated with a cgroup, + * the matching css from the cgroup's subsys table is guaranteed to + * be and stay valid until the enclosing operation is complete. + */ + if (cft->ss) + return rcu_dereference_raw(cgrp->subsys[cft->ss->id]); + else + return &cgrp->self; +} +EXPORT_SYMBOL_GPL(of_css); + +static int notify_on_release(const struct cgroup *cgrp) +{ + return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags); +} + +/** + * for_each_css - iterate all css's of a cgroup + * @css: the iteration cursor + * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end + * @cgrp: the target cgroup to iterate css's of + * + * Should be called under cgroup_[tree_]mutex. + */ +#define for_each_css(css, ssid, cgrp) \ + for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \ + if (!((css) = rcu_dereference_check( \ + (cgrp)->subsys[(ssid)], \ + lockdep_is_held(&cgroup_mutex)))) { } \ + else + +/** + * for_each_e_css - iterate all effective css's of a cgroup + * @css: the iteration cursor + * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end + * @cgrp: the target cgroup to iterate css's of + * + * Should be called under cgroup_[tree_]mutex. + */ +#define for_each_e_css(css, ssid, cgrp) \ + for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \ + if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \ + ; \ + else + +/** + * for_each_subsys - iterate all enabled cgroup subsystems + * @ss: the iteration cursor + * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end + */ +#define for_each_subsys(ss, ssid) \ + for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT && \ + (((ss) = cgroup_subsys[ssid]) || true); (ssid)++) + +/** + * do_each_subsys_mask - filter for_each_subsys with a bitmask + * @ss: the iteration cursor + * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end + * @ss_mask: the bitmask + * + * The block will only run for cases where the ssid-th bit (1 << ssid) of + * @ss_mask is set. + */ +#define do_each_subsys_mask(ss, ssid, ss_mask) do { \ + unsigned long __ss_mask = (ss_mask); \ + if (!CGROUP_SUBSYS_COUNT) { /* to avoid spurious gcc warning */ \ + (ssid) = 0; \ + break; \ + } \ + for_each_set_bit(ssid, &__ss_mask, CGROUP_SUBSYS_COUNT) { \ + (ss) = cgroup_subsys[ssid]; \ + { + +#define while_each_subsys_mask() \ + } \ + } \ +} while (false) + +/* iterate across the hierarchies */ +#define for_each_root(root) \ + list_for_each_entry((root), &cgroup_roots, root_list) + +/* iterate over child cgrps, lock should be held throughout iteration */ +#define cgroup_for_each_live_child(child, cgrp) \ + list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \ + if (({ lockdep_assert_held(&cgroup_mutex); \ + cgroup_is_dead(child); })) \ + ; \ + else + +/* walk live descendants in preorder */ +#define cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) \ + css_for_each_descendant_pre((d_css), cgroup_css((cgrp), NULL)) \ + if (({ lockdep_assert_held(&cgroup_mutex); \ + (dsct) = (d_css)->cgroup; \ + cgroup_is_dead(dsct); })) \ + ; \ + else + +/* walk live descendants in postorder */ +#define cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) \ + css_for_each_descendant_post((d_css), cgroup_css((cgrp), NULL)) \ + if (({ lockdep_assert_held(&cgroup_mutex); \ + (dsct) = (d_css)->cgroup; \ + cgroup_is_dead(dsct); })) \ + ; \ + else + +static void cgroup_release_agent(struct work_struct *work); +static void check_for_release(struct cgroup *cgrp); + +/* + * A cgroup can be associated with multiple css_sets as different tasks may + * belong to different cgroups on different hierarchies. In the other + * direction, a css_set is naturally associated with multiple cgroups. + * This M:N relationship is represented by the following link structure + * which exists for each association and allows traversing the associations + * from both sides. + */ +struct cgrp_cset_link { + /* the cgroup and css_set this link associates */ + struct cgroup *cgrp; + struct css_set *cset; + + /* list of cgrp_cset_links anchored at cgrp->cset_links */ + struct list_head cset_link; + + /* list of cgrp_cset_links anchored at css_set->cgrp_links */ + struct list_head cgrp_link; +}; + +/* + * The default css_set - used by init and its children prior to any + * hierarchies being mounted. It contains a pointer to the root state + * for each subsystem. Also used to anchor the list of css_sets. Not + * reference-counted, to improve performance when child cgroups + * haven't been created. + */ +struct css_set init_css_set = { + .refcount = ATOMIC_INIT(1), + .tasks = LIST_HEAD_INIT(init_css_set.tasks), + .mg_tasks = LIST_HEAD_INIT(init_css_set.mg_tasks), + .task_iters = LIST_HEAD_INIT(init_css_set.task_iters), + .cgrp_links = LIST_HEAD_INIT(init_css_set.cgrp_links), + .mg_preload_node = LIST_HEAD_INIT(init_css_set.mg_preload_node), + .mg_node = LIST_HEAD_INIT(init_css_set.mg_node), +}; + +static int css_set_count = 1; /* 1 for init_css_set */ + +/** + * css_set_populated - does a css_set contain any tasks? + * @cset: target css_set + */ +static bool css_set_populated(struct css_set *cset) +{ + lockdep_assert_held(&css_set_lock); + + return !list_empty(&cset->tasks) || !list_empty(&cset->mg_tasks); +} + +/** + * cgroup_update_populated - updated populated count of a cgroup + * @cgrp: the target cgroup + * @populated: inc or dec populated count + * + * One of the css_sets associated with @cgrp is either getting its first + * task or losing the last. Update @cgrp->populated_cnt accordingly. The + * count is propagated towards root so that a given cgroup's populated_cnt + * is zero iff the cgroup and all its descendants don't contain any tasks. + * + * @cgrp's interface file "cgroup.populated" is zero if + * @cgrp->populated_cnt is zero and 1 otherwise. When @cgrp->populated_cnt + * changes from or to zero, userland is notified that the content of the + * interface file has changed. This can be used to detect when @cgrp and + * its descendants become populated or empty. + */ +static void cgroup_update_populated(struct cgroup *cgrp, bool populated) +{ + lockdep_assert_held(&css_set_lock); + + do { + bool trigger; + + if (populated) + trigger = !cgrp->populated_cnt++; + else + trigger = !--cgrp->populated_cnt; + + if (!trigger) + break; + + check_for_release(cgrp); + cgroup_file_notify(&cgrp->events_file); + + cgrp = cgroup_parent(cgrp); + } while (cgrp); +} + +/** + * css_set_update_populated - update populated state of a css_set + * @cset: target css_set + * @populated: whether @cset is populated or depopulated + * + * @cset is either getting the first task or losing the last. Update the + * ->populated_cnt of all associated cgroups accordingly. + */ +static void css_set_update_populated(struct css_set *cset, bool populated) +{ + struct cgrp_cset_link *link; + + lockdep_assert_held(&css_set_lock); + + list_for_each_entry(link, &cset->cgrp_links, cgrp_link) + cgroup_update_populated(link->cgrp, populated); +} + +/** + * css_set_move_task - move a task from one css_set to another + * @task: task being moved + * @from_cset: css_set @task currently belongs to (may be NULL) + * @to_cset: new css_set @task is being moved to (may be NULL) + * @use_mg_tasks: move to @to_cset->mg_tasks instead of ->tasks + * + * Move @task from @from_cset to @to_cset. If @task didn't belong to any + * css_set, @from_cset can be NULL. If @task is being disassociated + * instead of moved, @to_cset can be NULL. + * + * This function automatically handles populated_cnt updates and + * css_task_iter adjustments but the caller is responsible for managing + * @from_cset and @to_cset's reference counts. + */ +static void css_set_move_task(struct task_struct *task, + struct css_set *from_cset, struct css_set *to_cset, + bool use_mg_tasks) +{ + lockdep_assert_held(&css_set_lock); + + if (to_cset && !css_set_populated(to_cset)) + css_set_update_populated(to_cset, true); + + if (from_cset) { + struct css_task_iter *it, *pos; + + WARN_ON_ONCE(list_empty(&task->cg_list)); + + /* + * @task is leaving, advance task iterators which are + * pointing to it so that they can resume at the next + * position. Advancing an iterator might remove it from + * the list, use safe walk. See css_task_iter_advance*() + * for details. + */ + list_for_each_entry_safe(it, pos, &from_cset->task_iters, + iters_node) + if (it->task_pos == &task->cg_list) + css_task_iter_advance(it); + + list_del_init(&task->cg_list); + if (!css_set_populated(from_cset)) + css_set_update_populated(from_cset, false); + } else { + WARN_ON_ONCE(!list_empty(&task->cg_list)); + } + + if (to_cset) { + /* + * We are synchronized through cgroup_threadgroup_rwsem + * against PF_EXITING setting such that we can't race + * against cgroup_exit() changing the css_set to + * init_css_set and dropping the old one. + */ + WARN_ON_ONCE(task->flags & PF_EXITING); + + rcu_assign_pointer(task->cgroups, to_cset); + list_add_tail(&task->cg_list, use_mg_tasks ? &to_cset->mg_tasks : + &to_cset->tasks); + } +} + +/* + * hash table for cgroup groups. This improves the performance to find + * an existing css_set. This hash doesn't (currently) take into + * account cgroups in empty hierarchies. + */ +#define CSS_SET_HASH_BITS 7 +static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS); + +static unsigned long css_set_hash(struct cgroup_subsys_state *css[]) +{ + unsigned long key = 0UL; + struct cgroup_subsys *ss; + int i; + + for_each_subsys(ss, i) + key += (unsigned long)css[i]; + key = (key >> 16) ^ key; + + return key; +} + +static void put_css_set_locked(struct css_set *cset) +{ + struct cgrp_cset_link *link, *tmp_link; + struct cgroup_subsys *ss; + int ssid; + + lockdep_assert_held(&css_set_lock); + + if (!atomic_dec_and_test(&cset->refcount)) + return; + + /* This css_set is dead. unlink it and release cgroup and css refs */ + for_each_subsys(ss, ssid) { + list_del(&cset->e_cset_node[ssid]); + css_put(cset->subsys[ssid]); + } + hash_del(&cset->hlist); + css_set_count--; + + list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) { + list_del(&link->cset_link); + list_del(&link->cgrp_link); + if (cgroup_parent(link->cgrp)) + cgroup_put(link->cgrp); + kfree(link); + } + + kfree_rcu(cset, rcu_head); +} + +static void put_css_set(struct css_set *cset) +{ + unsigned long flags; + + /* + * Ensure that the refcount doesn't hit zero while any readers + * can see it. Similar to atomic_dec_and_lock(), but for an + * rwlock + */ + if (atomic_add_unless(&cset->refcount, -1, 1)) + return; + + spin_lock_irqsave(&css_set_lock, flags); + put_css_set_locked(cset); + spin_unlock_irqrestore(&css_set_lock, flags); +} + +/* + * refcounted get/put for css_set objects + */ +static inline void get_css_set(struct css_set *cset) +{ + atomic_inc(&cset->refcount); +} + +/** + * compare_css_sets - helper function for find_existing_css_set(). + * @cset: candidate css_set being tested + * @old_cset: existing css_set for a task + * @new_cgrp: cgroup that's being entered by the task + * @template: desired set of css pointers in css_set (pre-calculated) + * + * Returns true if "cset" matches "old_cset" except for the hierarchy + * which "new_cgrp" belongs to, for which it should match "new_cgrp". + */ +static bool compare_css_sets(struct css_set *cset, + struct css_set *old_cset, + struct cgroup *new_cgrp, + struct cgroup_subsys_state *template[]) +{ + struct list_head *l1, *l2; + + /* + * On the default hierarchy, there can be csets which are + * associated with the same set of cgroups but different csses. + * Let's first ensure that csses match. + */ + if (memcmp(template, cset->subsys, sizeof(cset->subsys))) + return false; + + /* + * Compare cgroup pointers in order to distinguish between + * different cgroups in hierarchies. As different cgroups may + * share the same effective css, this comparison is always + * necessary. + */ + l1 = &cset->cgrp_links; + l2 = &old_cset->cgrp_links; + while (1) { + struct cgrp_cset_link *link1, *link2; + struct cgroup *cgrp1, *cgrp2; + + l1 = l1->next; + l2 = l2->next; + /* See if we reached the end - both lists are equal length. */ + if (l1 == &cset->cgrp_links) { + BUG_ON(l2 != &old_cset->cgrp_links); + break; + } else { + BUG_ON(l2 == &old_cset->cgrp_links); + } + /* Locate the cgroups associated with these links. */ + link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link); + link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link); + cgrp1 = link1->cgrp; + cgrp2 = link2->cgrp; + /* Hierarchies should be linked in the same order. */ + BUG_ON(cgrp1->root != cgrp2->root); + + /* + * If this hierarchy is the hierarchy of the cgroup + * that's changing, then we need to check that this + * css_set points to the new cgroup; if it's any other + * hierarchy, then this css_set should point to the + * same cgroup as the old css_set. + */ + if (cgrp1->root == new_cgrp->root) { + if (cgrp1 != new_cgrp) + return false; + } else { + if (cgrp1 != cgrp2) + return false; + } + } + return true; +} + +/** + * find_existing_css_set - init css array and find the matching css_set + * @old_cset: the css_set that we're using before the cgroup transition + * @cgrp: the cgroup that we're moving into + * @template: out param for the new set of csses, should be clear on entry + */ +static struct css_set *find_existing_css_set(struct css_set *old_cset, + struct cgroup *cgrp, + struct cgroup_subsys_state *template[]) +{ + struct cgroup_root *root = cgrp->root; + struct cgroup_subsys *ss; + struct css_set *cset; + unsigned long key; + int i; + + /* + * Build the set of subsystem state objects that we want to see in the + * new css_set. while subsystems can change globally, the entries here + * won't change, so no need for locking. + */ + for_each_subsys(ss, i) { + if (root->subsys_mask & (1UL << i)) { + /* + * @ss is in this hierarchy, so we want the + * effective css from @cgrp. + */ + template[i] = cgroup_e_css(cgrp, ss); + } else { + /* + * @ss is not in this hierarchy, so we don't want + * to change the css. + */ + template[i] = old_cset->subsys[i]; + } + } + + key = css_set_hash(template); + hash_for_each_possible(css_set_table, cset, hlist, key) { + if (!compare_css_sets(cset, old_cset, cgrp, template)) + continue; + + /* This css_set matches what we need */ + return cset; + } + + /* No existing cgroup group matched */ + return NULL; +} + +static void free_cgrp_cset_links(struct list_head *links_to_free) +{ + struct cgrp_cset_link *link, *tmp_link; + + list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) { + list_del(&link->cset_link); + kfree(link); + } +} + +/** + * allocate_cgrp_cset_links - allocate cgrp_cset_links + * @count: the number of links to allocate + * @tmp_links: list_head the allocated links are put on + * + * Allocate @count cgrp_cset_link structures and chain them on @tmp_links + * through ->cset_link. Returns 0 on success or -errno. + */ +static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links) +{ + struct cgrp_cset_link *link; + int i; + + INIT_LIST_HEAD(tmp_links); + + for (i = 0; i < count; i++) { + link = kzalloc(sizeof(*link), GFP_KERNEL); + if (!link) { + free_cgrp_cset_links(tmp_links); + return -ENOMEM; + } + list_add(&link->cset_link, tmp_links); + } + return 0; +} + +/** + * link_css_set - a helper function to link a css_set to a cgroup + * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links() + * @cset: the css_set to be linked + * @cgrp: the destination cgroup + */ +static void link_css_set(struct list_head *tmp_links, struct css_set *cset, + struct cgroup *cgrp) +{ + struct cgrp_cset_link *link; + + BUG_ON(list_empty(tmp_links)); + + if (cgroup_on_dfl(cgrp)) + cset->dfl_cgrp = cgrp; + + link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link); + link->cset = cset; + link->cgrp = cgrp; + + /* + * Always add links to the tail of the lists so that the lists are + * in choronological order. + */ + list_move_tail(&link->cset_link, &cgrp->cset_links); + list_add_tail(&link->cgrp_link, &cset->cgrp_links); + + if (cgroup_parent(cgrp)) + cgroup_get(cgrp); +} + +/** + * find_css_set - return a new css_set with one cgroup updated + * @old_cset: the baseline css_set + * @cgrp: the cgroup to be updated + * + * Return a new css_set that's equivalent to @old_cset, but with @cgrp + * substituted into the appropriate hierarchy. + */ +static struct css_set *find_css_set(struct css_set *old_cset, + struct cgroup *cgrp) +{ + struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { }; + struct css_set *cset; + struct list_head tmp_links; + struct cgrp_cset_link *link; + struct cgroup_subsys *ss; + unsigned long key; + int ssid; + + lockdep_assert_held(&cgroup_mutex); + + /* First see if we already have a cgroup group that matches + * the desired set */ + spin_lock_irq(&css_set_lock); + cset = find_existing_css_set(old_cset, cgrp, template); + if (cset) + get_css_set(cset); + spin_unlock_irq(&css_set_lock); + + if (cset) + return cset; + + cset = kzalloc(sizeof(*cset), GFP_KERNEL); + if (!cset) + return NULL; + + /* Allocate all the cgrp_cset_link objects that we'll need */ + if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) { + kfree(cset); + return NULL; + } + + atomic_set(&cset->refcount, 1); + INIT_LIST_HEAD(&cset->tasks); + INIT_LIST_HEAD(&cset->mg_tasks); + INIT_LIST_HEAD(&cset->task_iters); + INIT_HLIST_NODE(&cset->hlist); + INIT_LIST_HEAD(&cset->cgrp_links); + INIT_LIST_HEAD(&cset->mg_preload_node); + INIT_LIST_HEAD(&cset->mg_node); + + /* Copy the set of subsystem state objects generated in + * find_existing_css_set() */ + memcpy(cset->subsys, template, sizeof(cset->subsys)); + + spin_lock_irq(&css_set_lock); + /* Add reference counts and links from the new css_set. */ + list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) { + struct cgroup *c = link->cgrp; + + if (c->root == cgrp->root) + c = cgrp; + link_css_set(&tmp_links, cset, c); + } + + BUG_ON(!list_empty(&tmp_links)); + + css_set_count++; + + /* Add @cset to the hash table */ + key = css_set_hash(cset->subsys); + hash_add(css_set_table, &cset->hlist, key); + + for_each_subsys(ss, ssid) { + struct cgroup_subsys_state *css = cset->subsys[ssid]; + + list_add_tail(&cset->e_cset_node[ssid], + &css->cgroup->e_csets[ssid]); + css_get(css); + } + + spin_unlock_irq(&css_set_lock); + + return cset; +} + +static struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root) +{ + struct cgroup *root_cgrp = kf_root->kn->priv; + + return root_cgrp->root; +} + +static int cgroup_init_root_id(struct cgroup_root *root) +{ + int id; + + lockdep_assert_held(&cgroup_mutex); + + id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL); + if (id < 0) + return id; + + root->hierarchy_id = id; + return 0; +} + +static void cgroup_exit_root_id(struct cgroup_root *root) +{ + lockdep_assert_held(&cgroup_mutex); + + idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id); +} + +static void cgroup_free_root(struct cgroup_root *root) +{ + if (root) { + idr_destroy(&root->cgroup_idr); + kfree(root); + } +} + +static void cgroup_destroy_root(struct cgroup_root *root) +{ + struct cgroup *cgrp = &root->cgrp; + struct cgrp_cset_link *link, *tmp_link; + + trace_cgroup_destroy_root(root); + + cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); + + BUG_ON(atomic_read(&root->nr_cgrps)); + BUG_ON(!list_empty(&cgrp->self.children)); + + /* Rebind all subsystems back to the default hierarchy */ + WARN_ON(rebind_subsystems(&cgrp_dfl_root, root->subsys_mask)); + + /* + * Release all the links from cset_links to this hierarchy's + * root cgroup + */ + spin_lock_irq(&css_set_lock); + + list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) { + list_del(&link->cset_link); + list_del(&link->cgrp_link); + kfree(link); + } + + spin_unlock_irq(&css_set_lock); + + if (!list_empty(&root->root_list)) { + list_del(&root->root_list); + cgroup_root_count--; + } + + cgroup_exit_root_id(root); + + mutex_unlock(&cgroup_mutex); + + kernfs_destroy_root(root->kf_root); + cgroup_free_root(root); +} + +/* + * look up cgroup associated with current task's cgroup namespace on the + * specified hierarchy + */ +static struct cgroup * +current_cgns_cgroup_from_root(struct cgroup_root *root) +{ + struct cgroup *res = NULL; + struct css_set *cset; + + lockdep_assert_held(&css_set_lock); + + rcu_read_lock(); + + cset = current->nsproxy->cgroup_ns->root_cset; + if (cset == &init_css_set) { + res = &root->cgrp; + } else { + struct cgrp_cset_link *link; + + list_for_each_entry(link, &cset->cgrp_links, cgrp_link) { + struct cgroup *c = link->cgrp; + + if (c->root == root) { + res = c; + break; + } + } + } + rcu_read_unlock(); + + BUG_ON(!res); + return res; +} + +/* look up cgroup associated with given css_set on the specified hierarchy */ +static struct cgroup *cset_cgroup_from_root(struct css_set *cset, + struct cgroup_root *root) +{ + struct cgroup *res = NULL; + + lockdep_assert_held(&cgroup_mutex); + lockdep_assert_held(&css_set_lock); + + if (cset == &init_css_set) { + res = &root->cgrp; + } else { + struct cgrp_cset_link *link; + + list_for_each_entry(link, &cset->cgrp_links, cgrp_link) { + struct cgroup *c = link->cgrp; + + if (c->root == root) { + res = c; + break; + } + } + } + + BUG_ON(!res); + return res; +} + +/* + * Return the cgroup for "task" from the given hierarchy. Must be + * called with cgroup_mutex and css_set_lock held. + */ +static struct cgroup *task_cgroup_from_root(struct task_struct *task, + struct cgroup_root *root) +{ + /* + * No need to lock the task - since we hold cgroup_mutex the + * task can't change groups, so the only thing that can happen + * is that it exits and its css is set back to init_css_set. + */ + return cset_cgroup_from_root(task_css_set(task), root); +} + +/* + * A task must hold cgroup_mutex to modify cgroups. + * + * Any task can increment and decrement the count field without lock. + * So in general, code holding cgroup_mutex can't rely on the count + * field not changing. However, if the count goes to zero, then only + * cgroup_attach_task() can increment it again. Because a count of zero + * means that no tasks are currently attached, therefore there is no + * way a task attached to that cgroup can fork (the other way to + * increment the count). So code holding cgroup_mutex can safely + * assume that if the count is zero, it will stay zero. Similarly, if + * a task holds cgroup_mutex on a cgroup with zero count, it + * knows that the cgroup won't be removed, as cgroup_rmdir() + * needs that mutex. + * + * A cgroup can only be deleted if both its 'count' of using tasks + * is zero, and its list of 'children' cgroups is empty. Since all + * tasks in the system use _some_ cgroup, and since there is always at + * least one task in the system (init, pid == 1), therefore, root cgroup + * always has either children cgroups and/or using tasks. So we don't + * need a special hack to ensure that root cgroup cannot be deleted. + * + * P.S. One more locking exception. RCU is used to guard the + * update of a tasks cgroup pointer by cgroup_attach_task() + */ + +static struct kernfs_syscall_ops cgroup_kf_syscall_ops; +static const struct file_operations proc_cgroupstats_operations; + +static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft, + char *buf) +{ + struct cgroup_subsys *ss = cft->ss; + + if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) && + !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) + snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s", + cgroup_on_dfl(cgrp) ? ss->name : ss->legacy_name, + cft->name); + else + strncpy(buf, cft->name, CGROUP_FILE_NAME_MAX); + return buf; +} + +/** + * cgroup_file_mode - deduce file mode of a control file + * @cft: the control file in question + * + * S_IRUGO for read, S_IWUSR for write. + */ +static umode_t cgroup_file_mode(const struct cftype *cft) +{ + umode_t mode = 0; + + if (cft->read_u64 || cft->read_s64 || cft->seq_show) + mode |= S_IRUGO; + + if (cft->write_u64 || cft->write_s64 || cft->write) { + if (cft->flags & CFTYPE_WORLD_WRITABLE) + mode |= S_IWUGO; + else + mode |= S_IWUSR; + } + + return mode; +} + +/** + * cgroup_calc_subtree_ss_mask - calculate subtree_ss_mask + * @subtree_control: the new subtree_control mask to consider + * @this_ss_mask: available subsystems + * + * On the default hierarchy, a subsystem may request other subsystems to be + * enabled together through its ->depends_on mask. In such cases, more + * subsystems than specified in "cgroup.subtree_control" may be enabled. + * + * This function calculates which subsystems need to be enabled if + * @subtree_control is to be applied while restricted to @this_ss_mask. + */ +static u16 cgroup_calc_subtree_ss_mask(u16 subtree_control, u16 this_ss_mask) +{ + u16 cur_ss_mask = subtree_control; + struct cgroup_subsys *ss; + int ssid; + + lockdep_assert_held(&cgroup_mutex); + + cur_ss_mask |= cgrp_dfl_implicit_ss_mask; + + while (true) { + u16 new_ss_mask = cur_ss_mask; + + do_each_subsys_mask(ss, ssid, cur_ss_mask) { + new_ss_mask |= ss->depends_on; + } while_each_subsys_mask(); + + /* + * Mask out subsystems which aren't available. This can + * happen only if some depended-upon subsystems were bound + * to non-default hierarchies. + */ + new_ss_mask &= this_ss_mask; + + if (new_ss_mask == cur_ss_mask) + break; + cur_ss_mask = new_ss_mask; + } + + return cur_ss_mask; +} + +/** + * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods + * @kn: the kernfs_node being serviced + * + * This helper undoes cgroup_kn_lock_live() and should be invoked before + * the method finishes if locking succeeded. Note that once this function + * returns the cgroup returned by cgroup_kn_lock_live() may become + * inaccessible any time. If the caller intends to continue to access the + * cgroup, it should pin it before invoking this function. + */ +static void cgroup_kn_unlock(struct kernfs_node *kn) +{ + struct cgroup *cgrp; + + if (kernfs_type(kn) == KERNFS_DIR) + cgrp = kn->priv; + else + cgrp = kn->parent->priv; + + mutex_unlock(&cgroup_mutex); + + kernfs_unbreak_active_protection(kn); + cgroup_put(cgrp); +} + +/** + * cgroup_kn_lock_live - locking helper for cgroup kernfs methods + * @kn: the kernfs_node being serviced + * @drain_offline: perform offline draining on the cgroup + * + * This helper is to be used by a cgroup kernfs method currently servicing + * @kn. It breaks the active protection, performs cgroup locking and + * verifies that the associated cgroup is alive. Returns the cgroup if + * alive; otherwise, %NULL. A successful return should be undone by a + * matching cgroup_kn_unlock() invocation. If @drain_offline is %true, the + * cgroup is drained of offlining csses before return. + * + * Any cgroup kernfs method implementation which requires locking the + * associated cgroup should use this helper. It avoids nesting cgroup + * locking under kernfs active protection and allows all kernfs operations + * including self-removal. + */ +static struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn, + bool drain_offline) +{ + struct cgroup *cgrp; + + if (kernfs_type(kn) == KERNFS_DIR) + cgrp = kn->priv; + else + cgrp = kn->parent->priv; + + /* + * We're gonna grab cgroup_mutex which nests outside kernfs + * active_ref. cgroup liveliness check alone provides enough + * protection against removal. Ensure @cgrp stays accessible and + * break the active_ref protection. + */ + if (!cgroup_tryget(cgrp)) + return NULL; + kernfs_break_active_protection(kn); + + if (drain_offline) + cgroup_lock_and_drain_offline(cgrp); + else + mutex_lock(&cgroup_mutex); + + if (!cgroup_is_dead(cgrp)) + return cgrp; + + cgroup_kn_unlock(kn); + return NULL; +} + +static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft) +{ + char name[CGROUP_FILE_NAME_MAX]; + + lockdep_assert_held(&cgroup_mutex); + + if (cft->file_offset) { + struct cgroup_subsys_state *css = cgroup_css(cgrp, cft->ss); + struct cgroup_file *cfile = (void *)css + cft->file_offset; + + spin_lock_irq(&cgroup_file_kn_lock); + cfile->kn = NULL; + spin_unlock_irq(&cgroup_file_kn_lock); + } + + kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name)); +} + +/** + * css_clear_dir - remove subsys files in a cgroup directory + * @css: taget css + */ +static void css_clear_dir(struct cgroup_subsys_state *css) +{ + struct cgroup *cgrp = css->cgroup; + struct cftype *cfts; + + if (!(css->flags & CSS_VISIBLE)) + return; + + css->flags &= ~CSS_VISIBLE; + + list_for_each_entry(cfts, &css->ss->cfts, node) + cgroup_addrm_files(css, cgrp, cfts, false); +} + +/** + * css_populate_dir - create subsys files in a cgroup directory + * @css: target css + * + * On failure, no file is added. + */ +static int css_populate_dir(struct cgroup_subsys_state *css) +{ + struct cgroup *cgrp = css->cgroup; + struct cftype *cfts, *failed_cfts; + int ret; + + if ((css->flags & CSS_VISIBLE) || !cgrp->kn) + return 0; + + if (!css->ss) { + if (cgroup_on_dfl(cgrp)) + cfts = cgroup_dfl_base_files; + else + cfts = cgroup_legacy_base_files; + + return cgroup_addrm_files(&cgrp->self, cgrp, cfts, true); + } + + list_for_each_entry(cfts, &css->ss->cfts, node) { + ret = cgroup_addrm_files(css, cgrp, cfts, true); + if (ret < 0) { + failed_cfts = cfts; + goto err; + } + } + + css->flags |= CSS_VISIBLE; + + return 0; +err: + list_for_each_entry(cfts, &css->ss->cfts, node) { + if (cfts == failed_cfts) + break; + cgroup_addrm_files(css, cgrp, cfts, false); + } + return ret; +} + +static int rebind_subsystems(struct cgroup_root *dst_root, u16 ss_mask) +{ + struct cgroup *dcgrp = &dst_root->cgrp; + struct cgroup_subsys *ss; + int ssid, i, ret; + + lockdep_assert_held(&cgroup_mutex); + + do_each_subsys_mask(ss, ssid, ss_mask) { + /* + * If @ss has non-root csses attached to it, can't move. + * If @ss is an implicit controller, it is exempt from this + * rule and can be stolen. + */ + if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss)) && + !ss->implicit_on_dfl) + return -EBUSY; + + /* can't move between two non-dummy roots either */ + if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root) + return -EBUSY; + } while_each_subsys_mask(); + + do_each_subsys_mask(ss, ssid, ss_mask) { + struct cgroup_root *src_root = ss->root; + struct cgroup *scgrp = &src_root->cgrp; + struct cgroup_subsys_state *css = cgroup_css(scgrp, ss); + struct css_set *cset; + + WARN_ON(!css || cgroup_css(dcgrp, ss)); + + /* disable from the source */ + src_root->subsys_mask &= ~(1 << ssid); + WARN_ON(cgroup_apply_control(scgrp)); + cgroup_finalize_control(scgrp, 0); + + /* rebind */ + RCU_INIT_POINTER(scgrp->subsys[ssid], NULL); + rcu_assign_pointer(dcgrp->subsys[ssid], css); + ss->root = dst_root; + css->cgroup = dcgrp; + + spin_lock_irq(&css_set_lock); + hash_for_each(css_set_table, i, cset, hlist) + list_move_tail(&cset->e_cset_node[ss->id], + &dcgrp->e_csets[ss->id]); + spin_unlock_irq(&css_set_lock); + + /* default hierarchy doesn't enable controllers by default */ + dst_root->subsys_mask |= 1 << ssid; + if (dst_root == &cgrp_dfl_root) { + static_branch_enable(cgroup_subsys_on_dfl_key[ssid]); + } else { + dcgrp->subtree_control |= 1 << ssid; + static_branch_disable(cgroup_subsys_on_dfl_key[ssid]); + } + + ret = cgroup_apply_control(dcgrp); + if (ret) + pr_warn("partial failure to rebind %s controller (err=%d)\n", + ss->name, ret); + + if (ss->bind) + ss->bind(css); + } while_each_subsys_mask(); + + kernfs_activate(dcgrp->kn); + return 0; +} + +static int cgroup_show_path(struct seq_file *sf, struct kernfs_node *kf_node, + struct kernfs_root *kf_root) +{ + int len = 0; + char *buf = NULL; + struct cgroup_root *kf_cgroot = cgroup_root_from_kf(kf_root); + struct cgroup *ns_cgroup; + + buf = kmalloc(PATH_MAX, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + spin_lock_irq(&css_set_lock); + ns_cgroup = current_cgns_cgroup_from_root(kf_cgroot); + len = kernfs_path_from_node(kf_node, ns_cgroup->kn, buf, PATH_MAX); + spin_unlock_irq(&css_set_lock); + + if (len >= PATH_MAX) + len = -ERANGE; + else if (len > 0) { + seq_escape(sf, buf, " \t\n\\"); + len = 0; + } + kfree(buf); + return len; +} + +static int cgroup_show_options(struct seq_file *seq, + struct kernfs_root *kf_root) +{ + struct cgroup_root *root = cgroup_root_from_kf(kf_root); + struct cgroup_subsys *ss; + int ssid; + + if (root != &cgrp_dfl_root) + for_each_subsys(ss, ssid) + if (root->subsys_mask & (1 << ssid)) + seq_show_option(seq, ss->legacy_name, NULL); + if (root->flags & CGRP_ROOT_NOPREFIX) + seq_puts(seq, ",noprefix"); + if (root->flags & CGRP_ROOT_XATTR) + seq_puts(seq, ",xattr"); + + spin_lock(&release_agent_path_lock); + if (strlen(root->release_agent_path)) + seq_show_option(seq, "release_agent", + root->release_agent_path); + spin_unlock(&release_agent_path_lock); + + if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags)) + seq_puts(seq, ",clone_children"); + if (strlen(root->name)) + seq_show_option(seq, "name", root->name); + return 0; +} + +struct cgroup_sb_opts { + u16 subsys_mask; + unsigned int flags; + char *release_agent; + bool cpuset_clone_children; + char *name; + /* User explicitly requested empty subsystem */ + bool none; +}; + +static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts) +{ + char *token, *o = data; + bool all_ss = false, one_ss = false; + u16 mask = U16_MAX; + struct cgroup_subsys *ss; + int nr_opts = 0; + int i; + +#ifdef CONFIG_CPUSETS + mask = ~((u16)1 << cpuset_cgrp_id); +#endif + + memset(opts, 0, sizeof(*opts)); + + while ((token = strsep(&o, ",")) != NULL) { + nr_opts++; + + if (!*token) + return -EINVAL; + if (!strcmp(token, "none")) { + /* Explicitly have no subsystems */ + opts->none = true; + continue; + } + if (!strcmp(token, "all")) { + /* Mutually exclusive option 'all' + subsystem name */ + if (one_ss) + return -EINVAL; + all_ss = true; + continue; + } + if (!strcmp(token, "noprefix")) { + opts->flags |= CGRP_ROOT_NOPREFIX; + continue; + } + if (!strcmp(token, "clone_children")) { + opts->cpuset_clone_children = true; + continue; + } + if (!strcmp(token, "xattr")) { + opts->flags |= CGRP_ROOT_XATTR; + continue; + } + if (!strncmp(token, "release_agent=", 14)) { + /* Specifying two release agents is forbidden */ + if (opts->release_agent) + return -EINVAL; + opts->release_agent = + kstrndup(token + 14, PATH_MAX - 1, GFP_KERNEL); + if (!opts->release_agent) + return -ENOMEM; + continue; + } + if (!strncmp(token, "name=", 5)) { + const char *name = token + 5; + /* Can't specify an empty name */ + if (!strlen(name)) + return -EINVAL; + /* Must match [\w.-]+ */ + for (i = 0; i < strlen(name); i++) { + char c = name[i]; + if (isalnum(c)) + continue; + if ((c == '.') || (c == '-') || (c == '_')) + continue; + return -EINVAL; + } + /* Specifying two names is forbidden */ + if (opts->name) + return -EINVAL; + opts->name = kstrndup(name, + MAX_CGROUP_ROOT_NAMELEN - 1, + GFP_KERNEL); + if (!opts->name) + return -ENOMEM; + + continue; + } + + for_each_subsys(ss, i) { + if (strcmp(token, ss->legacy_name)) + continue; + if (!cgroup_ssid_enabled(i)) + continue; + if (cgroup_ssid_no_v1(i)) + continue; + + /* Mutually exclusive option 'all' + subsystem name */ + if (all_ss) + return -EINVAL; + opts->subsys_mask |= (1 << i); + one_ss = true; + + break; + } + if (i == CGROUP_SUBSYS_COUNT) + return -ENOENT; + } + + /* + * If the 'all' option was specified select all the subsystems, + * otherwise if 'none', 'name=' and a subsystem name options were + * not specified, let's default to 'all' + */ + if (all_ss || (!one_ss && !opts->none && !opts->name)) + for_each_subsys(ss, i) + if (cgroup_ssid_enabled(i) && !cgroup_ssid_no_v1(i)) + opts->subsys_mask |= (1 << i); + + /* + * We either have to specify by name or by subsystems. (So all + * empty hierarchies must have a name). + */ + if (!opts->subsys_mask && !opts->name) + return -EINVAL; + + /* + * Option noprefix was introduced just for backward compatibility + * with the old cpuset, so we allow noprefix only if mounting just + * the cpuset subsystem. + */ + if ((opts->flags & CGRP_ROOT_NOPREFIX) && (opts->subsys_mask & mask)) + return -EINVAL; + + /* Can't specify "none" and some subsystems */ + if (opts->subsys_mask && opts->none) + return -EINVAL; + + return 0; +} + +static int cgroup_remount(struct kernfs_root *kf_root, int *flags, char *data) +{ + int ret = 0; + struct cgroup_root *root = cgroup_root_from_kf(kf_root); + struct cgroup_sb_opts opts; + u16 added_mask, removed_mask; + + if (root == &cgrp_dfl_root) { + pr_err("remount is not allowed\n"); + return -EINVAL; + } + + cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); + + /* See what subsystems are wanted */ + ret = parse_cgroupfs_options(data, &opts); + if (ret) + goto out_unlock; + + if (opts.subsys_mask != root->subsys_mask || opts.release_agent) + pr_warn("option changes via remount are deprecated (pid=%d comm=%s)\n", + task_tgid_nr(current), current->comm); + + added_mask = opts.subsys_mask & ~root->subsys_mask; + removed_mask = root->subsys_mask & ~opts.subsys_mask; + + /* Don't allow flags or name to change at remount */ + if ((opts.flags ^ root->flags) || + (opts.name && strcmp(opts.name, root->name))) { + pr_err("option or name mismatch, new: 0x%x \"%s\", old: 0x%x \"%s\"\n", + opts.flags, opts.name ?: "", root->flags, root->name); + ret = -EINVAL; + goto out_unlock; + } + + /* remounting is not allowed for populated hierarchies */ + if (!list_empty(&root->cgrp.self.children)) { + ret = -EBUSY; + goto out_unlock; + } + + ret = rebind_subsystems(root, added_mask); + if (ret) + goto out_unlock; + + WARN_ON(rebind_subsystems(&cgrp_dfl_root, removed_mask)); + + if (opts.release_agent) { + spin_lock(&release_agent_path_lock); + strcpy(root->release_agent_path, opts.release_agent); + spin_unlock(&release_agent_path_lock); + } + + trace_cgroup_remount(root); + + out_unlock: + kfree(opts.release_agent); + kfree(opts.name); + mutex_unlock(&cgroup_mutex); + return ret; +} + +/* + * To reduce the fork() overhead for systems that are not actually using + * their cgroups capability, we don't maintain the lists running through + * each css_set to its tasks until we see the list actually used - in other + * words after the first mount. + */ +static bool use_task_css_set_links __read_mostly; + +static void cgroup_enable_task_cg_lists(void) +{ + struct task_struct *p, *g; + + spin_lock_irq(&css_set_lock); + + if (use_task_css_set_links) + goto out_unlock; + + use_task_css_set_links = true; + + /* + * We need tasklist_lock because RCU is not safe against + * while_each_thread(). Besides, a forking task that has passed + * cgroup_post_fork() without seeing use_task_css_set_links = 1 + * is not guaranteed to have its child immediately visible in the + * tasklist if we walk through it with RCU. + */ + read_lock(&tasklist_lock); + do_each_thread(g, p) { + WARN_ON_ONCE(!list_empty(&p->cg_list) || + task_css_set(p) != &init_css_set); + + /* + * We should check if the process is exiting, otherwise + * it will race with cgroup_exit() in that the list + * entry won't be deleted though the process has exited. + * Do it while holding siglock so that we don't end up + * racing against cgroup_exit(). + * + * Interrupts were already disabled while acquiring + * the css_set_lock, so we do not need to disable it + * again when acquiring the sighand->siglock here. + */ + spin_lock(&p->sighand->siglock); + if (!(p->flags & PF_EXITING)) { + struct css_set *cset = task_css_set(p); + + if (!css_set_populated(cset)) + css_set_update_populated(cset, true); + list_add_tail(&p->cg_list, &cset->tasks); + get_css_set(cset); + } + spin_unlock(&p->sighand->siglock); + } while_each_thread(g, p); + read_unlock(&tasklist_lock); +out_unlock: + spin_unlock_irq(&css_set_lock); +} + +static void init_cgroup_housekeeping(struct cgroup *cgrp) +{ + struct cgroup_subsys *ss; + int ssid; + + INIT_LIST_HEAD(&cgrp->self.sibling); + INIT_LIST_HEAD(&cgrp->self.children); + INIT_LIST_HEAD(&cgrp->cset_links); + INIT_LIST_HEAD(&cgrp->pidlists); + mutex_init(&cgrp->pidlist_mutex); + cgrp->self.cgroup = cgrp; + cgrp->self.flags |= CSS_ONLINE; + + for_each_subsys(ss, ssid) + INIT_LIST_HEAD(&cgrp->e_csets[ssid]); + + init_waitqueue_head(&cgrp->offline_waitq); + INIT_WORK(&cgrp->release_agent_work, cgroup_release_agent); +} + +static void init_cgroup_root(struct cgroup_root *root, + struct cgroup_sb_opts *opts) +{ + struct cgroup *cgrp = &root->cgrp; + + INIT_LIST_HEAD(&root->root_list); + atomic_set(&root->nr_cgrps, 1); + cgrp->root = root; + init_cgroup_housekeeping(cgrp); + idr_init(&root->cgroup_idr); + + root->flags = opts->flags; + if (opts->release_agent) + strcpy(root->release_agent_path, opts->release_agent); + if (opts->name) + strcpy(root->name, opts->name); + if (opts->cpuset_clone_children) + set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags); +} + +static int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask) +{ + LIST_HEAD(tmp_links); + struct cgroup *root_cgrp = &root->cgrp; + struct css_set *cset; + int i, ret; + + lockdep_assert_held(&cgroup_mutex); + + ret = cgroup_idr_alloc(&root->cgroup_idr, root_cgrp, 1, 2, GFP_KERNEL); + if (ret < 0) + goto out; + root_cgrp->id = ret; + root_cgrp->ancestor_ids[0] = ret; + + ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release, 0, + GFP_KERNEL); + if (ret) + goto out; + + /* + * We're accessing css_set_count without locking css_set_lock here, + * but that's OK - it can only be increased by someone holding + * cgroup_lock, and that's us. Later rebinding may disable + * controllers on the default hierarchy and thus create new csets, + * which can't be more than the existing ones. Allocate 2x. + */ + ret = allocate_cgrp_cset_links(2 * css_set_count, &tmp_links); + if (ret) + goto cancel_ref; + + ret = cgroup_init_root_id(root); + if (ret) + goto cancel_ref; + + root->kf_root = kernfs_create_root(&cgroup_kf_syscall_ops, + KERNFS_ROOT_CREATE_DEACTIVATED, + root_cgrp); + if (IS_ERR(root->kf_root)) { + ret = PTR_ERR(root->kf_root); + goto exit_root_id; + } + root_cgrp->kn = root->kf_root->kn; + + ret = css_populate_dir(&root_cgrp->self); + if (ret) + goto destroy_root; + + ret = rebind_subsystems(root, ss_mask); + if (ret) + goto destroy_root; + + trace_cgroup_setup_root(root); + + /* + * There must be no failure case after here, since rebinding takes + * care of subsystems' refcounts, which are explicitly dropped in + * the failure exit path. + */ + list_add(&root->root_list, &cgroup_roots); + cgroup_root_count++; + + /* + * Link the root cgroup in this hierarchy into all the css_set + * objects. + */ + spin_lock_irq(&css_set_lock); + hash_for_each(css_set_table, i, cset, hlist) { + link_css_set(&tmp_links, cset, root_cgrp); + if (css_set_populated(cset)) + cgroup_update_populated(root_cgrp, true); + } + spin_unlock_irq(&css_set_lock); + + BUG_ON(!list_empty(&root_cgrp->self.children)); + BUG_ON(atomic_read(&root->nr_cgrps) != 1); + + kernfs_activate(root_cgrp->kn); + ret = 0; + goto out; + +destroy_root: + kernfs_destroy_root(root->kf_root); + root->kf_root = NULL; +exit_root_id: + cgroup_exit_root_id(root); +cancel_ref: + percpu_ref_exit(&root_cgrp->self.refcnt); +out: + free_cgrp_cset_links(&tmp_links); + return ret; +} + +static struct dentry *cgroup_mount(struct file_system_type *fs_type, + int flags, const char *unused_dev_name, + void *data) +{ + bool is_v2 = fs_type == &cgroup2_fs_type; + struct super_block *pinned_sb = NULL; + struct cgroup_namespace *ns = current->nsproxy->cgroup_ns; + struct cgroup_subsys *ss; + struct cgroup_root *root; + struct cgroup_sb_opts opts; + struct dentry *dentry; + int ret; + int i; + bool new_sb; + + get_cgroup_ns(ns); + + /* Check if the caller has permission to mount. */ + if (!ns_capable(ns->user_ns, CAP_SYS_ADMIN)) { + put_cgroup_ns(ns); + return ERR_PTR(-EPERM); + } + + /* + * The first time anyone tries to mount a cgroup, enable the list + * linking each css_set to its tasks and fix up all existing tasks. + */ + if (!use_task_css_set_links) + cgroup_enable_task_cg_lists(); + + if (is_v2) { + if (data) { + pr_err("cgroup2: unknown option \"%s\"\n", (char *)data); + put_cgroup_ns(ns); + return ERR_PTR(-EINVAL); + } + cgrp_dfl_visible = true; + root = &cgrp_dfl_root; + cgroup_get(&root->cgrp); + goto out_mount; + } + + cgroup_lock_and_drain_offline(&cgrp_dfl_root.cgrp); + + /* First find the desired set of subsystems */ + ret = parse_cgroupfs_options(data, &opts); + if (ret) + goto out_unlock; + + /* + * Destruction of cgroup root is asynchronous, so subsystems may + * still be dying after the previous unmount. Let's drain the + * dying subsystems. We just need to ensure that the ones + * unmounted previously finish dying and don't care about new ones + * starting. Testing ref liveliness is good enough. + */ + for_each_subsys(ss, i) { + if (!(opts.subsys_mask & (1 << i)) || + ss->root == &cgrp_dfl_root) + continue; + + if (!percpu_ref_tryget_live(&ss->root->cgrp.self.refcnt)) { + mutex_unlock(&cgroup_mutex); + msleep(10); + ret = restart_syscall(); + goto out_free; + } + cgroup_put(&ss->root->cgrp); + } + + for_each_root(root) { + bool name_match = false; + + if (root == &cgrp_dfl_root) + continue; + + /* + * If we asked for a name then it must match. Also, if + * name matches but sybsys_mask doesn't, we should fail. + * Remember whether name matched. + */ + if (opts.name) { + if (strcmp(opts.name, root->name)) + continue; + name_match = true; + } + + /* + * If we asked for subsystems (or explicitly for no + * subsystems) then they must match. + */ + if ((opts.subsys_mask || opts.none) && + (opts.subsys_mask != root->subsys_mask)) { + if (!name_match) + continue; + ret = -EBUSY; + goto out_unlock; + } + + if (root->flags ^ opts.flags) + pr_warn("new mount options do not match the existing superblock, will be ignored\n"); + + /* + * We want to reuse @root whose lifetime is governed by its + * ->cgrp. Let's check whether @root is alive and keep it + * that way. As cgroup_kill_sb() can happen anytime, we + * want to block it by pinning the sb so that @root doesn't + * get killed before mount is complete. + * + * With the sb pinned, tryget_live can reliably indicate + * whether @root can be reused. If it's being killed, + * drain it. We can use wait_queue for the wait but this + * path is super cold. Let's just sleep a bit and retry. + */ + pinned_sb = kernfs_pin_sb(root->kf_root, NULL); + if (IS_ERR(pinned_sb) || + !percpu_ref_tryget_live(&root->cgrp.self.refcnt)) { + mutex_unlock(&cgroup_mutex); + if (!IS_ERR_OR_NULL(pinned_sb)) + deactivate_super(pinned_sb); + msleep(10); + ret = restart_syscall(); + goto out_free; + } + + ret = 0; + goto out_unlock; + } + + /* + * No such thing, create a new one. name= matching without subsys + * specification is allowed for already existing hierarchies but we + * can't create new one without subsys specification. + */ + if (!opts.subsys_mask && !opts.none) { + ret = -EINVAL; + goto out_unlock; + } + + /* Hierarchies may only be created in the initial cgroup namespace. */ + if (ns != &init_cgroup_ns) { + ret = -EPERM; + goto out_unlock; + } + + root = kzalloc(sizeof(*root), GFP_KERNEL); + if (!root) { + ret = -ENOMEM; + goto out_unlock; + } + + init_cgroup_root(root, &opts); + + ret = cgroup_setup_root(root, opts.subsys_mask); + if (ret) + cgroup_free_root(root); + +out_unlock: + mutex_unlock(&cgroup_mutex); +out_free: + kfree(opts.release_agent); + kfree(opts.name); + + if (ret) { + put_cgroup_ns(ns); + return ERR_PTR(ret); + } +out_mount: + dentry = kernfs_mount(fs_type, flags, root->kf_root, + is_v2 ? CGROUP2_SUPER_MAGIC : CGROUP_SUPER_MAGIC, + &new_sb); + + /* + * In non-init cgroup namespace, instead of root cgroup's + * dentry, we return the dentry corresponding to the + * cgroupns->root_cgrp. + */ + if (!IS_ERR(dentry) && ns != &init_cgroup_ns) { + struct dentry *nsdentry; + struct cgroup *cgrp; + + mutex_lock(&cgroup_mutex); + spin_lock_irq(&css_set_lock); + + cgrp = cset_cgroup_from_root(ns->root_cset, root); + + spin_unlock_irq(&css_set_lock); + mutex_unlock(&cgroup_mutex); + + nsdentry = kernfs_node_dentry(cgrp->kn, dentry->d_sb); + dput(dentry); + dentry = nsdentry; + } + + if (IS_ERR(dentry) || !new_sb) + cgroup_put(&root->cgrp); + + /* + * If @pinned_sb, we're reusing an existing root and holding an + * extra ref on its sb. Mount is complete. Put the extra ref. + */ + if (pinned_sb) { + WARN_ON(new_sb); + deactivate_super(pinned_sb); + } + + put_cgroup_ns(ns); + return dentry; +} + +static void cgroup_kill_sb(struct super_block *sb) +{ + struct kernfs_root *kf_root = kernfs_root_from_sb(sb); + struct cgroup_root *root = cgroup_root_from_kf(kf_root); + + /* + * If @root doesn't have any mounts or children, start killing it. + * This prevents new mounts by disabling percpu_ref_tryget_live(). + * cgroup_mount() may wait for @root's release. + * + * And don't kill the default root. + */ + if (!list_empty(&root->cgrp.self.children) || + root == &cgrp_dfl_root) + cgroup_put(&root->cgrp); + else + percpu_ref_kill(&root->cgrp.self.refcnt); + + kernfs_kill_sb(sb); +} + +static struct file_system_type cgroup_fs_type = { + .name = "cgroup", + .mount = cgroup_mount, + .kill_sb = cgroup_kill_sb, + .fs_flags = FS_USERNS_MOUNT, +}; + +static struct file_system_type cgroup2_fs_type = { + .name = "cgroup2", + .mount = cgroup_mount, + .kill_sb = cgroup_kill_sb, + .fs_flags = FS_USERNS_MOUNT, +}; + +static int cgroup_path_ns_locked(struct cgroup *cgrp, char *buf, size_t buflen, + struct cgroup_namespace *ns) +{ + struct cgroup *root = cset_cgroup_from_root(ns->root_cset, cgrp->root); + + return kernfs_path_from_node(cgrp->kn, root->kn, buf, buflen); +} + +int cgroup_path_ns(struct cgroup *cgrp, char *buf, size_t buflen, + struct cgroup_namespace *ns) +{ + int ret; + + mutex_lock(&cgroup_mutex); + spin_lock_irq(&css_set_lock); + + ret = cgroup_path_ns_locked(cgrp, buf, buflen, ns); + + spin_unlock_irq(&css_set_lock); + mutex_unlock(&cgroup_mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(cgroup_path_ns); + +/** + * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy + * @task: target task + * @buf: the buffer to write the path into + * @buflen: the length of the buffer + * + * Determine @task's cgroup on the first (the one with the lowest non-zero + * hierarchy_id) cgroup hierarchy and copy its path into @buf. This + * function grabs cgroup_mutex and shouldn't be used inside locks used by + * cgroup controller callbacks. + * + * Return value is the same as kernfs_path(). + */ +int task_cgroup_path(struct task_struct *task, char *buf, size_t buflen) +{ + struct cgroup_root *root; + struct cgroup *cgrp; + int hierarchy_id = 1; + int ret; + + mutex_lock(&cgroup_mutex); + spin_lock_irq(&css_set_lock); + + root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id); + + if (root) { + cgrp = task_cgroup_from_root(task, root); + ret = cgroup_path_ns_locked(cgrp, buf, buflen, &init_cgroup_ns); + } else { + /* if no hierarchy exists, everyone is in "/" */ + ret = strlcpy(buf, "/", buflen); + } + + spin_unlock_irq(&css_set_lock); + mutex_unlock(&cgroup_mutex); + return ret; +} +EXPORT_SYMBOL_GPL(task_cgroup_path); + +/* used to track tasks and other necessary states during migration */ +struct cgroup_taskset { + /* the src and dst cset list running through cset->mg_node */ + struct list_head src_csets; + struct list_head dst_csets; + + /* the subsys currently being processed */ + int ssid; + + /* + * Fields for cgroup_taskset_*() iteration. + * + * Before migration is committed, the target migration tasks are on + * ->mg_tasks of the csets on ->src_csets. After, on ->mg_tasks of + * the csets on ->dst_csets. ->csets point to either ->src_csets + * or ->dst_csets depending on whether migration is committed. + * + * ->cur_csets and ->cur_task point to the current task position + * during iteration. + */ + struct list_head *csets; + struct css_set *cur_cset; + struct task_struct *cur_task; +}; + +#define CGROUP_TASKSET_INIT(tset) (struct cgroup_taskset){ \ + .src_csets = LIST_HEAD_INIT(tset.src_csets), \ + .dst_csets = LIST_HEAD_INIT(tset.dst_csets), \ + .csets = &tset.src_csets, \ +} + +/** + * cgroup_taskset_add - try to add a migration target task to a taskset + * @task: target task + * @tset: target taskset + * + * Add @task, which is a migration target, to @tset. This function becomes + * noop if @task doesn't need to be migrated. @task's css_set should have + * been added as a migration source and @task->cg_list will be moved from + * the css_set's tasks list to mg_tasks one. + */ +static void cgroup_taskset_add(struct task_struct *task, + struct cgroup_taskset *tset) +{ + struct css_set *cset; + + lockdep_assert_held(&css_set_lock); + + /* @task either already exited or can't exit until the end */ + if (task->flags & PF_EXITING) + return; + + /* leave @task alone if post_fork() hasn't linked it yet */ + if (list_empty(&task->cg_list)) + return; + + cset = task_css_set(task); + if (!cset->mg_src_cgrp) + return; + + list_move_tail(&task->cg_list, &cset->mg_tasks); + if (list_empty(&cset->mg_node)) + list_add_tail(&cset->mg_node, &tset->src_csets); + if (list_empty(&cset->mg_dst_cset->mg_node)) + list_move_tail(&cset->mg_dst_cset->mg_node, + &tset->dst_csets); +} + +/** + * cgroup_taskset_first - reset taskset and return the first task + * @tset: taskset of interest + * @dst_cssp: output variable for the destination css + * + * @tset iteration is initialized and the first task is returned. + */ +struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset, + struct cgroup_subsys_state **dst_cssp) +{ + tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node); + tset->cur_task = NULL; + + return cgroup_taskset_next(tset, dst_cssp); +} + +/** + * cgroup_taskset_next - iterate to the next task in taskset + * @tset: taskset of interest + * @dst_cssp: output variable for the destination css + * + * Return the next task in @tset. Iteration must have been initialized + * with cgroup_taskset_first(). + */ +struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset, + struct cgroup_subsys_state **dst_cssp) +{ + struct css_set *cset = tset->cur_cset; + struct task_struct *task = tset->cur_task; + + while (&cset->mg_node != tset->csets) { + if (!task) + task = list_first_entry(&cset->mg_tasks, + struct task_struct, cg_list); + else + task = list_next_entry(task, cg_list); + + if (&task->cg_list != &cset->mg_tasks) { + tset->cur_cset = cset; + tset->cur_task = task; + + /* + * This function may be called both before and + * after cgroup_taskset_migrate(). The two cases + * can be distinguished by looking at whether @cset + * has its ->mg_dst_cset set. + */ + if (cset->mg_dst_cset) + *dst_cssp = cset->mg_dst_cset->subsys[tset->ssid]; + else + *dst_cssp = cset->subsys[tset->ssid]; + + return task; + } + + cset = list_next_entry(cset, mg_node); + task = NULL; + } + + return NULL; +} + +/** + * cgroup_taskset_migrate - migrate a taskset + * @tset: taget taskset + * @root: cgroup root the migration is taking place on + * + * Migrate tasks in @tset as setup by migration preparation functions. + * This function fails iff one of the ->can_attach callbacks fails and + * guarantees that either all or none of the tasks in @tset are migrated. + * @tset is consumed regardless of success. + */ +static int cgroup_taskset_migrate(struct cgroup_taskset *tset, + struct cgroup_root *root) +{ + struct cgroup_subsys *ss; + struct task_struct *task, *tmp_task; + struct css_set *cset, *tmp_cset; + int ssid, failed_ssid, ret; + + /* methods shouldn't be called if no task is actually migrating */ + if (list_empty(&tset->src_csets)) + return 0; + + /* check that we can legitimately attach to the cgroup */ + do_each_subsys_mask(ss, ssid, root->subsys_mask) { + if (ss->can_attach) { + tset->ssid = ssid; + ret = ss->can_attach(tset); + if (ret) { + failed_ssid = ssid; + goto out_cancel_attach; + } + } + } while_each_subsys_mask(); + + /* + * Now that we're guaranteed success, proceed to move all tasks to + * the new cgroup. There are no failure cases after here, so this + * is the commit point. + */ + spin_lock_irq(&css_set_lock); + list_for_each_entry(cset, &tset->src_csets, mg_node) { + list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) { + struct css_set *from_cset = task_css_set(task); + struct css_set *to_cset = cset->mg_dst_cset; + + get_css_set(to_cset); + css_set_move_task(task, from_cset, to_cset, true); + put_css_set_locked(from_cset); + } + } + spin_unlock_irq(&css_set_lock); + + /* + * Migration is committed, all target tasks are now on dst_csets. + * Nothing is sensitive to fork() after this point. Notify + * controllers that migration is complete. + */ + tset->csets = &tset->dst_csets; + + do_each_subsys_mask(ss, ssid, root->subsys_mask) { + if (ss->attach) { + tset->ssid = ssid; + ss->attach(tset); + } + } while_each_subsys_mask(); + + ret = 0; + goto out_release_tset; + +out_cancel_attach: + do_each_subsys_mask(ss, ssid, root->subsys_mask) { + if (ssid == failed_ssid) + break; + if (ss->cancel_attach) { + tset->ssid = ssid; + ss->cancel_attach(tset); + } + } while_each_subsys_mask(); +out_release_tset: + spin_lock_irq(&css_set_lock); + list_splice_init(&tset->dst_csets, &tset->src_csets); + list_for_each_entry_safe(cset, tmp_cset, &tset->src_csets, mg_node) { + list_splice_tail_init(&cset->mg_tasks, &cset->tasks); + list_del_init(&cset->mg_node); + } + spin_unlock_irq(&css_set_lock); + return ret; +} + +/** + * cgroup_may_migrate_to - verify whether a cgroup can be migration destination + * @dst_cgrp: destination cgroup to test + * + * On the default hierarchy, except for the root, subtree_control must be + * zero for migration destination cgroups with tasks so that child cgroups + * don't compete against tasks. + */ +static bool cgroup_may_migrate_to(struct cgroup *dst_cgrp) +{ + return !cgroup_on_dfl(dst_cgrp) || !cgroup_parent(dst_cgrp) || + !dst_cgrp->subtree_control; +} + +/** + * cgroup_migrate_finish - cleanup after attach + * @preloaded_csets: list of preloaded css_sets + * + * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst(). See + * those functions for details. + */ +static void cgroup_migrate_finish(struct list_head *preloaded_csets) +{ + struct css_set *cset, *tmp_cset; + + lockdep_assert_held(&cgroup_mutex); + + spin_lock_irq(&css_set_lock); + list_for_each_entry_safe(cset, tmp_cset, preloaded_csets, mg_preload_node) { + cset->mg_src_cgrp = NULL; + cset->mg_dst_cgrp = NULL; + cset->mg_dst_cset = NULL; + list_del_init(&cset->mg_preload_node); + put_css_set_locked(cset); + } + spin_unlock_irq(&css_set_lock); +} + +/** + * cgroup_migrate_add_src - add a migration source css_set + * @src_cset: the source css_set to add + * @dst_cgrp: the destination cgroup + * @preloaded_csets: list of preloaded css_sets + * + * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp. Pin + * @src_cset and add it to @preloaded_csets, which should later be cleaned + * up by cgroup_migrate_finish(). + * + * This function may be called without holding cgroup_threadgroup_rwsem + * even if the target is a process. Threads may be created and destroyed + * but as long as cgroup_mutex is not dropped, no new css_set can be put + * into play and the preloaded css_sets are guaranteed to cover all + * migrations. + */ +static void cgroup_migrate_add_src(struct css_set *src_cset, + struct cgroup *dst_cgrp, + struct list_head *preloaded_csets) +{ + struct cgroup *src_cgrp; + + lockdep_assert_held(&cgroup_mutex); + lockdep_assert_held(&css_set_lock); + + /* + * If ->dead, @src_set is associated with one or more dead cgroups + * and doesn't contain any migratable tasks. Ignore it early so + * that the rest of migration path doesn't get confused by it. + */ + if (src_cset->dead) + return; + + src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root); + + if (!list_empty(&src_cset->mg_preload_node)) + return; + + WARN_ON(src_cset->mg_src_cgrp); + WARN_ON(src_cset->mg_dst_cgrp); + WARN_ON(!list_empty(&src_cset->mg_tasks)); + WARN_ON(!list_empty(&src_cset->mg_node)); + + src_cset->mg_src_cgrp = src_cgrp; + src_cset->mg_dst_cgrp = dst_cgrp; + get_css_set(src_cset); + list_add(&src_cset->mg_preload_node, preloaded_csets); +} + +/** + * cgroup_migrate_prepare_dst - prepare destination css_sets for migration + * @preloaded_csets: list of preloaded source css_sets + * + * Tasks are about to be moved and all the source css_sets have been + * preloaded to @preloaded_csets. This function looks up and pins all + * destination css_sets, links each to its source, and append them to + * @preloaded_csets. + * + * This function must be called after cgroup_migrate_add_src() has been + * called on each migration source css_set. After migration is performed + * using cgroup_migrate(), cgroup_migrate_finish() must be called on + * @preloaded_csets. + */ +static int cgroup_migrate_prepare_dst(struct list_head *preloaded_csets) +{ + LIST_HEAD(csets); + struct css_set *src_cset, *tmp_cset; + + lockdep_assert_held(&cgroup_mutex); + + /* look up the dst cset for each src cset and link it to src */ + list_for_each_entry_safe(src_cset, tmp_cset, preloaded_csets, mg_preload_node) { + struct css_set *dst_cset; + + dst_cset = find_css_set(src_cset, src_cset->mg_dst_cgrp); + if (!dst_cset) + goto err; + + WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset); + + /* + * If src cset equals dst, it's noop. Drop the src. + * cgroup_migrate() will skip the cset too. Note that we + * can't handle src == dst as some nodes are used by both. + */ + if (src_cset == dst_cset) { + src_cset->mg_src_cgrp = NULL; + src_cset->mg_dst_cgrp = NULL; + list_del_init(&src_cset->mg_preload_node); + put_css_set(src_cset); + put_css_set(dst_cset); + continue; + } + + src_cset->mg_dst_cset = dst_cset; + + if (list_empty(&dst_cset->mg_preload_node)) + list_add(&dst_cset->mg_preload_node, &csets); + else + put_css_set(dst_cset); + } + + list_splice_tail(&csets, preloaded_csets); + return 0; +err: + cgroup_migrate_finish(&csets); + return -ENOMEM; +} + +/** + * cgroup_migrate - migrate a process or task to a cgroup + * @leader: the leader of the process or the task to migrate + * @threadgroup: whether @leader points to the whole process or a single task + * @root: cgroup root migration is taking place on + * + * Migrate a process or task denoted by @leader. If migrating a process, + * the caller must be holding cgroup_threadgroup_rwsem. The caller is also + * responsible for invoking cgroup_migrate_add_src() and + * cgroup_migrate_prepare_dst() on the targets before invoking this + * function and following up with cgroup_migrate_finish(). + * + * As long as a controller's ->can_attach() doesn't fail, this function is + * guaranteed to succeed. This means that, excluding ->can_attach() + * failure, when migrating multiple targets, the success or failure can be + * decided for all targets by invoking group_migrate_prepare_dst() before + * actually starting migrating. + */ +static int cgroup_migrate(struct task_struct *leader, bool threadgroup, + struct cgroup_root *root) +{ + struct cgroup_taskset tset = CGROUP_TASKSET_INIT(tset); + struct task_struct *task; + + /* + * Prevent freeing of tasks while we take a snapshot. Tasks that are + * already PF_EXITING could be freed from underneath us unless we + * take an rcu_read_lock. + */ + spin_lock_irq(&css_set_lock); + rcu_read_lock(); + task = leader; + do { + cgroup_taskset_add(task, &tset); + if (!threadgroup) + break; + } while_each_thread(leader, task); + rcu_read_unlock(); + spin_unlock_irq(&css_set_lock); + + return cgroup_taskset_migrate(&tset, root); +} + +/** + * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup + * @dst_cgrp: the cgroup to attach to + * @leader: the task or the leader of the threadgroup to be attached + * @threadgroup: attach the whole threadgroup? + * + * Call holding cgroup_mutex and cgroup_threadgroup_rwsem. + */ +static int cgroup_attach_task(struct cgroup *dst_cgrp, + struct task_struct *leader, bool threadgroup) +{ + LIST_HEAD(preloaded_csets); + struct task_struct *task; + int ret; + + if (!cgroup_may_migrate_to(dst_cgrp)) + return -EBUSY; + + /* look up all src csets */ + spin_lock_irq(&css_set_lock); + rcu_read_lock(); + task = leader; + do { + cgroup_migrate_add_src(task_css_set(task), dst_cgrp, + &preloaded_csets); + if (!threadgroup) + break; + } while_each_thread(leader, task); + rcu_read_unlock(); + spin_unlock_irq(&css_set_lock); + + /* prepare dst csets and commit */ + ret = cgroup_migrate_prepare_dst(&preloaded_csets); + if (!ret) + ret = cgroup_migrate(leader, threadgroup, dst_cgrp->root); + + cgroup_migrate_finish(&preloaded_csets); + + if (!ret) + trace_cgroup_attach_task(dst_cgrp, leader, threadgroup); + + return ret; +} + +static int cgroup_procs_write_permission(struct task_struct *task, + struct cgroup *dst_cgrp, + struct kernfs_open_file *of) +{ + const struct cred *cred = current_cred(); + const struct cred *tcred = get_task_cred(task); + int ret = 0; + + /* + * even if we're attaching all tasks in the thread group, we only + * need to check permissions on one of them. + */ + if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) && + !uid_eq(cred->euid, tcred->uid) && + !uid_eq(cred->euid, tcred->suid)) + ret = -EACCES; + + if (!ret && cgroup_on_dfl(dst_cgrp)) { + struct super_block *sb = of->file->f_path.dentry->d_sb; + struct cgroup *cgrp; + struct inode *inode; + + spin_lock_irq(&css_set_lock); + cgrp = task_cgroup_from_root(task, &cgrp_dfl_root); + spin_unlock_irq(&css_set_lock); + + while (!cgroup_is_descendant(dst_cgrp, cgrp)) + cgrp = cgroup_parent(cgrp); + + ret = -ENOMEM; + inode = kernfs_get_inode(sb, cgrp->procs_file.kn); + if (inode) { + ret = inode_permission(inode, MAY_WRITE); + iput(inode); + } + } + + put_cred(tcred); + return ret; +} + +/* + * Find the task_struct of the task to attach by vpid and pass it along to the + * function to attach either it or all tasks in its threadgroup. Will lock + * cgroup_mutex and threadgroup. + */ +static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off, bool threadgroup) +{ + struct task_struct *tsk; + struct cgroup_subsys *ss; + struct cgroup *cgrp; + pid_t pid; + int ssid, ret; + + if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0) + return -EINVAL; + + cgrp = cgroup_kn_lock_live(of->kn, false); + if (!cgrp) + return -ENODEV; + + percpu_down_write(&cgroup_threadgroup_rwsem); + rcu_read_lock(); + if (pid) { + tsk = find_task_by_vpid(pid); + if (!tsk) { + ret = -ESRCH; + goto out_unlock_rcu; + } + } else { + tsk = current; + } + + if (threadgroup) + tsk = tsk->group_leader; + + /* + * Workqueue threads may acquire PF_NO_SETAFFINITY and become + * trapped in a cpuset, or RT worker may be born in a cgroup + * with no rt_runtime allocated. Just say no. + */ + if (tsk == kthreadd_task || (tsk->flags & PF_NO_SETAFFINITY)) { + ret = -EINVAL; + goto out_unlock_rcu; + } + + get_task_struct(tsk); + rcu_read_unlock(); + + ret = cgroup_procs_write_permission(tsk, cgrp, of); + if (!ret) + ret = cgroup_attach_task(cgrp, tsk, threadgroup); + + put_task_struct(tsk); + goto out_unlock_threadgroup; + +out_unlock_rcu: + rcu_read_unlock(); +out_unlock_threadgroup: + percpu_up_write(&cgroup_threadgroup_rwsem); + for_each_subsys(ss, ssid) + if (ss->post_attach) + ss->post_attach(); + cgroup_kn_unlock(of->kn); + return ret ?: nbytes; +} + +/** + * cgroup_attach_task_all - attach task 'tsk' to all cgroups of task 'from' + * @from: attach to all cgroups of a given task + * @tsk: the task to be attached + */ +int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk) +{ + struct cgroup_root *root; + int retval = 0; + + mutex_lock(&cgroup_mutex); + percpu_down_write(&cgroup_threadgroup_rwsem); + for_each_root(root) { + struct cgroup *from_cgrp; + + if (root == &cgrp_dfl_root) + continue; + + spin_lock_irq(&css_set_lock); + from_cgrp = task_cgroup_from_root(from, root); + spin_unlock_irq(&css_set_lock); + + retval = cgroup_attach_task(from_cgrp, tsk, false); + if (retval) + break; + } + percpu_up_write(&cgroup_threadgroup_rwsem); + mutex_unlock(&cgroup_mutex); + + return retval; +} +EXPORT_SYMBOL_GPL(cgroup_attach_task_all); + +static ssize_t cgroup_tasks_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + return __cgroup_procs_write(of, buf, nbytes, off, false); +} + +static ssize_t cgroup_procs_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + return __cgroup_procs_write(of, buf, nbytes, off, true); +} + +static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct cgroup *cgrp; + + BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX); + + cgrp = cgroup_kn_lock_live(of->kn, false); + if (!cgrp) + return -ENODEV; + spin_lock(&release_agent_path_lock); + strlcpy(cgrp->root->release_agent_path, strstrip(buf), + sizeof(cgrp->root->release_agent_path)); + spin_unlock(&release_agent_path_lock); + cgroup_kn_unlock(of->kn); + return nbytes; +} + +static int cgroup_release_agent_show(struct seq_file *seq, void *v) +{ + struct cgroup *cgrp = seq_css(seq)->cgroup; + + spin_lock(&release_agent_path_lock); + seq_puts(seq, cgrp->root->release_agent_path); + spin_unlock(&release_agent_path_lock); + seq_putc(seq, '\n'); + return 0; +} + +static int cgroup_sane_behavior_show(struct seq_file *seq, void *v) +{ + seq_puts(seq, "0\n"); + return 0; +} + +static void cgroup_print_ss_mask(struct seq_file *seq, u16 ss_mask) +{ + struct cgroup_subsys *ss; + bool printed = false; + int ssid; + + do_each_subsys_mask(ss, ssid, ss_mask) { + if (printed) + seq_putc(seq, ' '); + seq_printf(seq, "%s", ss->name); + printed = true; + } while_each_subsys_mask(); + if (printed) + seq_putc(seq, '\n'); +} + +/* show controllers which are enabled from the parent */ +static int cgroup_controllers_show(struct seq_file *seq, void *v) +{ + struct cgroup *cgrp = seq_css(seq)->cgroup; + + cgroup_print_ss_mask(seq, cgroup_control(cgrp)); + return 0; +} + +/* show controllers which are enabled for a given cgroup's children */ +static int cgroup_subtree_control_show(struct seq_file *seq, void *v) +{ + struct cgroup *cgrp = seq_css(seq)->cgroup; + + cgroup_print_ss_mask(seq, cgrp->subtree_control); + return 0; +} + +/** + * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy + * @cgrp: root of the subtree to update csses for + * + * @cgrp's control masks have changed and its subtree's css associations + * need to be updated accordingly. This function looks up all css_sets + * which are attached to the subtree, creates the matching updated css_sets + * and migrates the tasks to the new ones. + */ +static int cgroup_update_dfl_csses(struct cgroup *cgrp) +{ + LIST_HEAD(preloaded_csets); + struct cgroup_taskset tset = CGROUP_TASKSET_INIT(tset); + struct cgroup_subsys_state *d_css; + struct cgroup *dsct; + struct css_set *src_cset; + int ret; + + lockdep_assert_held(&cgroup_mutex); + + percpu_down_write(&cgroup_threadgroup_rwsem); + + /* look up all csses currently attached to @cgrp's subtree */ + spin_lock_irq(&css_set_lock); + cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { + struct cgrp_cset_link *link; + + list_for_each_entry(link, &dsct->cset_links, cset_link) + cgroup_migrate_add_src(link->cset, dsct, + &preloaded_csets); + } + spin_unlock_irq(&css_set_lock); + + /* NULL dst indicates self on default hierarchy */ + ret = cgroup_migrate_prepare_dst(&preloaded_csets); + if (ret) + goto out_finish; + + spin_lock_irq(&css_set_lock); + list_for_each_entry(src_cset, &preloaded_csets, mg_preload_node) { + struct task_struct *task, *ntask; + + /* src_csets precede dst_csets, break on the first dst_cset */ + if (!src_cset->mg_src_cgrp) + break; + + /* all tasks in src_csets need to be migrated */ + list_for_each_entry_safe(task, ntask, &src_cset->tasks, cg_list) + cgroup_taskset_add(task, &tset); + } + spin_unlock_irq(&css_set_lock); + + ret = cgroup_taskset_migrate(&tset, cgrp->root); +out_finish: + cgroup_migrate_finish(&preloaded_csets); + percpu_up_write(&cgroup_threadgroup_rwsem); + return ret; +} + +/** + * cgroup_lock_and_drain_offline - lock cgroup_mutex and drain offlined csses + * @cgrp: root of the target subtree + * + * Because css offlining is asynchronous, userland may try to re-enable a + * controller while the previous css is still around. This function grabs + * cgroup_mutex and drains the previous css instances of @cgrp's subtree. + */ +static void cgroup_lock_and_drain_offline(struct cgroup *cgrp) + __acquires(&cgroup_mutex) +{ + struct cgroup *dsct; + struct cgroup_subsys_state *d_css; + struct cgroup_subsys *ss; + int ssid; + +restart: + mutex_lock(&cgroup_mutex); + + cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { + for_each_subsys(ss, ssid) { + struct cgroup_subsys_state *css = cgroup_css(dsct, ss); + DEFINE_WAIT(wait); + + if (!css || !percpu_ref_is_dying(&css->refcnt)) + continue; + + cgroup_get(dsct); + prepare_to_wait(&dsct->offline_waitq, &wait, + TASK_UNINTERRUPTIBLE); + + mutex_unlock(&cgroup_mutex); + schedule(); + finish_wait(&dsct->offline_waitq, &wait); + + cgroup_put(dsct); + goto restart; + } + } +} + +/** + * cgroup_save_control - save control masks of a subtree + * @cgrp: root of the target subtree + * + * Save ->subtree_control and ->subtree_ss_mask to the respective old_ + * prefixed fields for @cgrp's subtree including @cgrp itself. + */ +static void cgroup_save_control(struct cgroup *cgrp) +{ + struct cgroup *dsct; + struct cgroup_subsys_state *d_css; + + cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { + dsct->old_subtree_control = dsct->subtree_control; + dsct->old_subtree_ss_mask = dsct->subtree_ss_mask; + } +} + +/** + * cgroup_propagate_control - refresh control masks of a subtree + * @cgrp: root of the target subtree + * + * For @cgrp and its subtree, ensure ->subtree_ss_mask matches + * ->subtree_control and propagate controller availability through the + * subtree so that descendants don't have unavailable controllers enabled. + */ +static void cgroup_propagate_control(struct cgroup *cgrp) +{ + struct cgroup *dsct; + struct cgroup_subsys_state *d_css; + + cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { + dsct->subtree_control &= cgroup_control(dsct); + dsct->subtree_ss_mask = + cgroup_calc_subtree_ss_mask(dsct->subtree_control, + cgroup_ss_mask(dsct)); + } +} + +/** + * cgroup_restore_control - restore control masks of a subtree + * @cgrp: root of the target subtree + * + * Restore ->subtree_control and ->subtree_ss_mask from the respective old_ + * prefixed fields for @cgrp's subtree including @cgrp itself. + */ +static void cgroup_restore_control(struct cgroup *cgrp) +{ + struct cgroup *dsct; + struct cgroup_subsys_state *d_css; + + cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { + dsct->subtree_control = dsct->old_subtree_control; + dsct->subtree_ss_mask = dsct->old_subtree_ss_mask; + } +} + +static bool css_visible(struct cgroup_subsys_state *css) +{ + struct cgroup_subsys *ss = css->ss; + struct cgroup *cgrp = css->cgroup; + + if (cgroup_control(cgrp) & (1 << ss->id)) + return true; + if (!(cgroup_ss_mask(cgrp) & (1 << ss->id))) + return false; + return cgroup_on_dfl(cgrp) && ss->implicit_on_dfl; +} + +/** + * cgroup_apply_control_enable - enable or show csses according to control + * @cgrp: root of the target subtree + * + * Walk @cgrp's subtree and create new csses or make the existing ones + * visible. A css is created invisible if it's being implicitly enabled + * through dependency. An invisible css is made visible when the userland + * explicitly enables it. + * + * Returns 0 on success, -errno on failure. On failure, csses which have + * been processed already aren't cleaned up. The caller is responsible for + * cleaning up with cgroup_apply_control_disble(). + */ +static int cgroup_apply_control_enable(struct cgroup *cgrp) +{ + struct cgroup *dsct; + struct cgroup_subsys_state *d_css; + struct cgroup_subsys *ss; + int ssid, ret; + + cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) { + for_each_subsys(ss, ssid) { + struct cgroup_subsys_state *css = cgroup_css(dsct, ss); + + WARN_ON_ONCE(css && percpu_ref_is_dying(&css->refcnt)); + + if (!(cgroup_ss_mask(dsct) & (1 << ss->id))) + continue; + + if (!css) { + css = css_create(dsct, ss); + if (IS_ERR(css)) + return PTR_ERR(css); + } + + if (css_visible(css)) { + ret = css_populate_dir(css); + if (ret) + return ret; + } + } + } + + return 0; +} + +/** + * cgroup_apply_control_disable - kill or hide csses according to control + * @cgrp: root of the target subtree + * + * Walk @cgrp's subtree and kill and hide csses so that they match + * cgroup_ss_mask() and cgroup_visible_mask(). + * + * A css is hidden when the userland requests it to be disabled while other + * subsystems are still depending on it. The css must not actively control + * resources and be in the vanilla state if it's made visible again later. + * Controllers which may be depended upon should provide ->css_reset() for + * this purpose. + */ +static void cgroup_apply_control_disable(struct cgroup *cgrp) +{ + struct cgroup *dsct; + struct cgroup_subsys_state *d_css; + struct cgroup_subsys *ss; + int ssid; + + cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) { + for_each_subsys(ss, ssid) { + struct cgroup_subsys_state *css = cgroup_css(dsct, ss); + + WARN_ON_ONCE(css && percpu_ref_is_dying(&css->refcnt)); + + if (!css) + continue; + + if (css->parent && + !(cgroup_ss_mask(dsct) & (1 << ss->id))) { + kill_css(css); + } else if (!css_visible(css)) { + css_clear_dir(css); + if (ss->css_reset) + ss->css_reset(css); + } + } + } +} + +/** + * cgroup_apply_control - apply control mask updates to the subtree + * @cgrp: root of the target subtree + * + * subsystems can be enabled and disabled in a subtree using the following + * steps. + * + * 1. Call cgroup_save_control() to stash the current state. + * 2. Update ->subtree_control masks in the subtree as desired. + * 3. Call cgroup_apply_control() to apply the changes. + * 4. Optionally perform other related operations. + * 5. Call cgroup_finalize_control() to finish up. + * + * This function implements step 3 and propagates the mask changes + * throughout @cgrp's subtree, updates csses accordingly and perform + * process migrations. + */ +static int cgroup_apply_control(struct cgroup *cgrp) +{ + int ret; + + cgroup_propagate_control(cgrp); + + ret = cgroup_apply_control_enable(cgrp); + if (ret) + return ret; + + /* + * At this point, cgroup_e_css() results reflect the new csses + * making the following cgroup_update_dfl_csses() properly update + * css associations of all tasks in the subtree. + */ + ret = cgroup_update_dfl_csses(cgrp); + if (ret) + return ret; + + return 0; +} + +/** + * cgroup_finalize_control - finalize control mask update + * @cgrp: root of the target subtree + * @ret: the result of the update + * + * Finalize control mask update. See cgroup_apply_control() for more info. + */ +static void cgroup_finalize_control(struct cgroup *cgrp, int ret) +{ + if (ret) { + cgroup_restore_control(cgrp); + cgroup_propagate_control(cgrp); + } + + cgroup_apply_control_disable(cgrp); +} + +/* change the enabled child controllers for a cgroup in the default hierarchy */ +static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, + loff_t off) +{ + u16 enable = 0, disable = 0; + struct cgroup *cgrp, *child; + struct cgroup_subsys *ss; + char *tok; + int ssid, ret; + + /* + * Parse input - space separated list of subsystem names prefixed + * with either + or -. + */ + buf = strstrip(buf); + while ((tok = strsep(&buf, " "))) { + if (tok[0] == '\0') + continue; + do_each_subsys_mask(ss, ssid, ~cgrp_dfl_inhibit_ss_mask) { + if (!cgroup_ssid_enabled(ssid) || + strcmp(tok + 1, ss->name)) + continue; + + if (*tok == '+') { + enable |= 1 << ssid; + disable &= ~(1 << ssid); + } else if (*tok == '-') { + disable |= 1 << ssid; + enable &= ~(1 << ssid); + } else { + return -EINVAL; + } + break; + } while_each_subsys_mask(); + if (ssid == CGROUP_SUBSYS_COUNT) + return -EINVAL; + } + + cgrp = cgroup_kn_lock_live(of->kn, true); + if (!cgrp) + return -ENODEV; + + for_each_subsys(ss, ssid) { + if (enable & (1 << ssid)) { + if (cgrp->subtree_control & (1 << ssid)) { + enable &= ~(1 << ssid); + continue; + } + + if (!(cgroup_control(cgrp) & (1 << ssid))) { + ret = -ENOENT; + goto out_unlock; + } + } else if (disable & (1 << ssid)) { + if (!(cgrp->subtree_control & (1 << ssid))) { + disable &= ~(1 << ssid); + continue; + } + + /* a child has it enabled? */ + cgroup_for_each_live_child(child, cgrp) { + if (child->subtree_control & (1 << ssid)) { + ret = -EBUSY; + goto out_unlock; + } + } + } + } + + if (!enable && !disable) { + ret = 0; + goto out_unlock; + } + + /* + * Except for the root, subtree_control must be zero for a cgroup + * with tasks so that child cgroups don't compete against tasks. + */ + if (enable && cgroup_parent(cgrp)) { + struct cgrp_cset_link *link; + + /* + * Because namespaces pin csets too, @cgrp->cset_links + * might not be empty even when @cgrp is empty. Walk and + * verify each cset. + */ + spin_lock_irq(&css_set_lock); + + ret = 0; + list_for_each_entry(link, &cgrp->cset_links, cset_link) { + if (css_set_populated(link->cset)) { + ret = -EBUSY; + break; + } + } + + spin_unlock_irq(&css_set_lock); + + if (ret) + goto out_unlock; + } + + /* save and update control masks and prepare csses */ + cgroup_save_control(cgrp); + + cgrp->subtree_control |= enable; + cgrp->subtree_control &= ~disable; + + ret = cgroup_apply_control(cgrp); + + cgroup_finalize_control(cgrp, ret); + + kernfs_activate(cgrp->kn); + ret = 0; +out_unlock: + cgroup_kn_unlock(of->kn); + return ret ?: nbytes; +} + +static int cgroup_events_show(struct seq_file *seq, void *v) +{ + seq_printf(seq, "populated %d\n", + cgroup_is_populated(seq_css(seq)->cgroup)); + return 0; +} + +static int cgroup_file_open(struct kernfs_open_file *of) +{ + struct cftype *cft = of->kn->priv; + + if (cft->open) + return cft->open(of); + return 0; +} + +static void cgroup_file_release(struct kernfs_open_file *of) +{ + struct cftype *cft = of->kn->priv; + + if (cft->release) + cft->release(of); +} + +static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off) +{ + struct cgroup *cgrp = of->kn->parent->priv; + struct cftype *cft = of->kn->priv; + struct cgroup_subsys_state *css; + int ret; + + if (cft->write) + return cft->write(of, buf, nbytes, off); + + /* + * kernfs guarantees that a file isn't deleted with operations in + * flight, which means that the matching css is and stays alive and + * doesn't need to be pinned. The RCU locking is not necessary + * either. It's just for the convenience of using cgroup_css(). + */ + rcu_read_lock(); + css = cgroup_css(cgrp, cft->ss); + rcu_read_unlock(); + + if (cft->write_u64) { + unsigned long long v; + ret = kstrtoull(buf, 0, &v); + if (!ret) + ret = cft->write_u64(css, cft, v); + } else if (cft->write_s64) { + long long v; + ret = kstrtoll(buf, 0, &v); + if (!ret) + ret = cft->write_s64(css, cft, v); + } else { + ret = -EINVAL; + } + + return ret ?: nbytes; +} + +static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos) +{ + return seq_cft(seq)->seq_start(seq, ppos); +} + +static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos) +{ + return seq_cft(seq)->seq_next(seq, v, ppos); +} + +static void cgroup_seqfile_stop(struct seq_file *seq, void *v) +{ + if (seq_cft(seq)->seq_stop) + seq_cft(seq)->seq_stop(seq, v); +} + +static int cgroup_seqfile_show(struct seq_file *m, void *arg) +{ + struct cftype *cft = seq_cft(m); + struct cgroup_subsys_state *css = seq_css(m); + + if (cft->seq_show) + return cft->seq_show(m, arg); + + if (cft->read_u64) + seq_printf(m, "%llu\n", cft->read_u64(css, cft)); + else if (cft->read_s64) + seq_printf(m, "%lld\n", cft->read_s64(css, cft)); + else + return -EINVAL; + return 0; +} + +static struct kernfs_ops cgroup_kf_single_ops = { + .atomic_write_len = PAGE_SIZE, + .open = cgroup_file_open, + .release = cgroup_file_release, + .write = cgroup_file_write, + .seq_show = cgroup_seqfile_show, +}; + +static struct kernfs_ops cgroup_kf_ops = { + .atomic_write_len = PAGE_SIZE, + .open = cgroup_file_open, + .release = cgroup_file_release, + .write = cgroup_file_write, + .seq_start = cgroup_seqfile_start, + .seq_next = cgroup_seqfile_next, + .seq_stop = cgroup_seqfile_stop, + .seq_show = cgroup_seqfile_show, +}; + +/* + * cgroup_rename - Only allow simple rename of directories in place. + */ +static int cgroup_rename(struct kernfs_node *kn, struct kernfs_node *new_parent, + const char *new_name_str) +{ + struct cgroup *cgrp = kn->priv; + int ret; + + if (kernfs_type(kn) != KERNFS_DIR) + return -ENOTDIR; + if (kn->parent != new_parent) + return -EIO; + + /* + * This isn't a proper migration and its usefulness is very + * limited. Disallow on the default hierarchy. + */ + if (cgroup_on_dfl(cgrp)) + return -EPERM; + + /* + * We're gonna grab cgroup_mutex which nests outside kernfs + * active_ref. kernfs_rename() doesn't require active_ref + * protection. Break them before grabbing cgroup_mutex. + */ + kernfs_break_active_protection(new_parent); + kernfs_break_active_protection(kn); + + mutex_lock(&cgroup_mutex); + + ret = kernfs_rename(kn, new_parent, new_name_str); + if (!ret) + trace_cgroup_rename(cgrp); + + mutex_unlock(&cgroup_mutex); + + kernfs_unbreak_active_protection(kn); + kernfs_unbreak_active_protection(new_parent); + return ret; +} + +/* set uid and gid of cgroup dirs and files to that of the creator */ +static int cgroup_kn_set_ugid(struct kernfs_node *kn) +{ + struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID, + .ia_uid = current_fsuid(), + .ia_gid = current_fsgid(), }; + + if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) && + gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID)) + return 0; + + return kernfs_setattr(kn, &iattr); +} + +static int cgroup_add_file(struct cgroup_subsys_state *css, struct cgroup *cgrp, + struct cftype *cft) +{ + char name[CGROUP_FILE_NAME_MAX]; + struct kernfs_node *kn; + struct lock_class_key *key = NULL; + int ret; + +#ifdef CONFIG_DEBUG_LOCK_ALLOC + key = &cft->lockdep_key; +#endif + kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name), + cgroup_file_mode(cft), 0, cft->kf_ops, cft, + NULL, key); + if (IS_ERR(kn)) + return PTR_ERR(kn); + + ret = cgroup_kn_set_ugid(kn); + if (ret) { + kernfs_remove(kn); + return ret; + } + + if (cft->file_offset) { + struct cgroup_file *cfile = (void *)css + cft->file_offset; + + spin_lock_irq(&cgroup_file_kn_lock); + cfile->kn = kn; + spin_unlock_irq(&cgroup_file_kn_lock); + } + + return 0; +} + +/** + * cgroup_addrm_files - add or remove files to a cgroup directory + * @css: the target css + * @cgrp: the target cgroup (usually css->cgroup) + * @cfts: array of cftypes to be added + * @is_add: whether to add or remove + * + * Depending on @is_add, add or remove files defined by @cfts on @cgrp. + * For removals, this function never fails. + */ +static int cgroup_addrm_files(struct cgroup_subsys_state *css, + struct cgroup *cgrp, struct cftype cfts[], + bool is_add) +{ + struct cftype *cft, *cft_end = NULL; + int ret = 0; + + lockdep_assert_held(&cgroup_mutex); + +restart: + for (cft = cfts; cft != cft_end && cft->name[0] != '\0'; cft++) { + /* does cft->flags tell us to skip this file on @cgrp? */ + if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp)) + continue; + if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp)) + continue; + if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp)) + continue; + if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp)) + continue; + + if (is_add) { + ret = cgroup_add_file(css, cgrp, cft); + if (ret) { + pr_warn("%s: failed to add %s, err=%d\n", + __func__, cft->name, ret); + cft_end = cft; + is_add = false; + goto restart; + } + } else { + cgroup_rm_file(cgrp, cft); + } + } + return ret; +} + +static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add) +{ + LIST_HEAD(pending); + struct cgroup_subsys *ss = cfts[0].ss; + struct cgroup *root = &ss->root->cgrp; + struct cgroup_subsys_state *css; + int ret = 0; + + lockdep_assert_held(&cgroup_mutex); + + /* add/rm files for all cgroups created before */ + css_for_each_descendant_pre(css, cgroup_css(root, ss)) { + struct cgroup *cgrp = css->cgroup; + + if (!(css->flags & CSS_VISIBLE)) + continue; + + ret = cgroup_addrm_files(css, cgrp, cfts, is_add); + if (ret) + break; + } + + if (is_add && !ret) + kernfs_activate(root->kn); + return ret; +} + +static void cgroup_exit_cftypes(struct cftype *cfts) +{ + struct cftype *cft; + + for (cft = cfts; cft->name[0] != '\0'; cft++) { + /* free copy for custom atomic_write_len, see init_cftypes() */ + if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) + kfree(cft->kf_ops); + cft->kf_ops = NULL; + cft->ss = NULL; + + /* revert flags set by cgroup core while adding @cfts */ + cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL); + } +} + +static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) +{ + struct cftype *cft; + + for (cft = cfts; cft->name[0] != '\0'; cft++) { + struct kernfs_ops *kf_ops; + + WARN_ON(cft->ss || cft->kf_ops); + + if (cft->seq_start) + kf_ops = &cgroup_kf_ops; + else + kf_ops = &cgroup_kf_single_ops; + + /* + * Ugh... if @cft wants a custom max_write_len, we need to + * make a copy of kf_ops to set its atomic_write_len. + */ + if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) { + kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL); + if (!kf_ops) { + cgroup_exit_cftypes(cfts); + return -ENOMEM; + } + kf_ops->atomic_write_len = cft->max_write_len; + } + + cft->kf_ops = kf_ops; + cft->ss = ss; + } + + return 0; +} + +static int cgroup_rm_cftypes_locked(struct cftype *cfts) +{ + lockdep_assert_held(&cgroup_mutex); + + if (!cfts || !cfts[0].ss) + return -ENOENT; + + list_del(&cfts->node); + cgroup_apply_cftypes(cfts, false); + cgroup_exit_cftypes(cfts); + return 0; +} + +/** + * cgroup_rm_cftypes - remove an array of cftypes from a subsystem + * @cfts: zero-length name terminated array of cftypes + * + * Unregister @cfts. Files described by @cfts are removed from all + * existing cgroups and all future cgroups won't have them either. This + * function can be called anytime whether @cfts' subsys is attached or not. + * + * Returns 0 on successful unregistration, -ENOENT if @cfts is not + * registered. + */ +int cgroup_rm_cftypes(struct cftype *cfts) +{ + int ret; + + mutex_lock(&cgroup_mutex); + ret = cgroup_rm_cftypes_locked(cfts); + mutex_unlock(&cgroup_mutex); + return ret; +} + +/** + * cgroup_add_cftypes - add an array of cftypes to a subsystem + * @ss: target cgroup subsystem + * @cfts: zero-length name terminated array of cftypes + * + * Register @cfts to @ss. Files described by @cfts are created for all + * existing cgroups to which @ss is attached and all future cgroups will + * have them too. This function can be called anytime whether @ss is + * attached or not. + * + * Returns 0 on successful registration, -errno on failure. Note that this + * function currently returns 0 as long as @cfts registration is successful + * even if some file creation attempts on existing cgroups fail. + */ +static int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) +{ + int ret; + + if (!cgroup_ssid_enabled(ss->id)) + return 0; + + if (!cfts || cfts[0].name[0] == '\0') + return 0; + + ret = cgroup_init_cftypes(ss, cfts); + if (ret) + return ret; + + mutex_lock(&cgroup_mutex); + + list_add_tail(&cfts->node, &ss->cfts); + ret = cgroup_apply_cftypes(cfts, true); + if (ret) + cgroup_rm_cftypes_locked(cfts); + + mutex_unlock(&cgroup_mutex); + return ret; +} + +/** + * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy + * @ss: target cgroup subsystem + * @cfts: zero-length name terminated array of cftypes + * + * Similar to cgroup_add_cftypes() but the added files are only used for + * the default hierarchy. + */ +int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) +{ + struct cftype *cft; + + for (cft = cfts; cft && cft->name[0] != '\0'; cft++) + cft->flags |= __CFTYPE_ONLY_ON_DFL; + return cgroup_add_cftypes(ss, cfts); +} + +/** + * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies + * @ss: target cgroup subsystem + * @cfts: zero-length name terminated array of cftypes + * + * Similar to cgroup_add_cftypes() but the added files are only used for + * the legacy hierarchies. + */ +int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) +{ + struct cftype *cft; + + for (cft = cfts; cft && cft->name[0] != '\0'; cft++) + cft->flags |= __CFTYPE_NOT_ON_DFL; + return cgroup_add_cftypes(ss, cfts); +} + +/** + * cgroup_file_notify - generate a file modified event for a cgroup_file + * @cfile: target cgroup_file + * + * @cfile must have been obtained by setting cftype->file_offset. + */ +void cgroup_file_notify(struct cgroup_file *cfile) +{ + unsigned long flags; + + spin_lock_irqsave(&cgroup_file_kn_lock, flags); + if (cfile->kn) + kernfs_notify(cfile->kn); + spin_unlock_irqrestore(&cgroup_file_kn_lock, flags); +} + +/** + * cgroup_task_count - count the number of tasks in a cgroup. + * @cgrp: the cgroup in question + * + * Return the number of tasks in the cgroup. The returned number can be + * higher than the actual number of tasks due to css_set references from + * namespace roots and temporary usages. + */ +static int cgroup_task_count(const struct cgroup *cgrp) +{ + int count = 0; + struct cgrp_cset_link *link; + + spin_lock_irq(&css_set_lock); + list_for_each_entry(link, &cgrp->cset_links, cset_link) + count += atomic_read(&link->cset->refcount); + spin_unlock_irq(&css_set_lock); + return count; +} + +/** + * css_next_child - find the next child of a given css + * @pos: the current position (%NULL to initiate traversal) + * @parent: css whose children to walk + * + * This function returns the next child of @parent and should be called + * under either cgroup_mutex or RCU read lock. The only requirement is + * that @parent and @pos are accessible. The next sibling is guaranteed to + * be returned regardless of their states. + * + * If a subsystem synchronizes ->css_online() and the start of iteration, a + * css which finished ->css_online() is guaranteed to be visible in the + * future iterations and will stay visible until the last reference is put. + * A css which hasn't finished ->css_online() or already finished + * ->css_offline() may show up during traversal. It's each subsystem's + * responsibility to synchronize against on/offlining. + */ +struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos, + struct cgroup_subsys_state *parent) +{ + struct cgroup_subsys_state *next; + + cgroup_assert_mutex_or_rcu_locked(); + + /* + * @pos could already have been unlinked from the sibling list. + * Once a cgroup is removed, its ->sibling.next is no longer + * updated when its next sibling changes. CSS_RELEASED is set when + * @pos is taken off list, at which time its next pointer is valid, + * and, as releases are serialized, the one pointed to by the next + * pointer is guaranteed to not have started release yet. This + * implies that if we observe !CSS_RELEASED on @pos in this RCU + * critical section, the one pointed to by its next pointer is + * guaranteed to not have finished its RCU grace period even if we + * have dropped rcu_read_lock() inbetween iterations. + * + * If @pos has CSS_RELEASED set, its next pointer can't be + * dereferenced; however, as each css is given a monotonically + * increasing unique serial number and always appended to the + * sibling list, the next one can be found by walking the parent's + * children until the first css with higher serial number than + * @pos's. While this path can be slower, it happens iff iteration + * races against release and the race window is very small. + */ + if (!pos) { + next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling); + } else if (likely(!(pos->flags & CSS_RELEASED))) { + next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling); + } else { + list_for_each_entry_rcu(next, &parent->children, sibling) + if (next->serial_nr > pos->serial_nr) + break; + } + + /* + * @next, if not pointing to the head, can be dereferenced and is + * the next sibling. + */ + if (&next->sibling != &parent->children) + return next; + return NULL; +} + +/** + * css_next_descendant_pre - find the next descendant for pre-order walk + * @pos: the current position (%NULL to initiate traversal) + * @root: css whose descendants to walk + * + * To be used by css_for_each_descendant_pre(). Find the next descendant + * to visit for pre-order traversal of @root's descendants. @root is + * included in the iteration and the first node to be visited. + * + * While this function requires cgroup_mutex or RCU read locking, it + * doesn't require the whole traversal to be contained in a single critical + * section. This function will return the correct next descendant as long + * as both @pos and @root are accessible and @pos is a descendant of @root. + * + * If a subsystem synchronizes ->css_online() and the start of iteration, a + * css which finished ->css_online() is guaranteed to be visible in the + * future iterations and will stay visible until the last reference is put. + * A css which hasn't finished ->css_online() or already finished + * ->css_offline() may show up during traversal. It's each subsystem's + * responsibility to synchronize against on/offlining. + */ +struct cgroup_subsys_state * +css_next_descendant_pre(struct cgroup_subsys_state *pos, + struct cgroup_subsys_state *root) +{ + struct cgroup_subsys_state *next; + + cgroup_assert_mutex_or_rcu_locked(); + + /* if first iteration, visit @root */ + if (!pos) + return root; + + /* visit the first child if exists */ + next = css_next_child(NULL, pos); + if (next) + return next; + + /* no child, visit my or the closest ancestor's next sibling */ + while (pos != root) { + next = css_next_child(pos, pos->parent); + if (next) + return next; + pos = pos->parent; + } + + return NULL; +} + +/** + * css_rightmost_descendant - return the rightmost descendant of a css + * @pos: css of interest + * + * Return the rightmost descendant of @pos. If there's no descendant, @pos + * is returned. This can be used during pre-order traversal to skip + * subtree of @pos. + * + * While this function requires cgroup_mutex or RCU read locking, it + * doesn't require the whole traversal to be contained in a single critical + * section. This function will return the correct rightmost descendant as + * long as @pos is accessible. + */ +struct cgroup_subsys_state * +css_rightmost_descendant(struct cgroup_subsys_state *pos) +{ + struct cgroup_subsys_state *last, *tmp; + + cgroup_assert_mutex_or_rcu_locked(); + + do { + last = pos; + /* ->prev isn't RCU safe, walk ->next till the end */ + pos = NULL; + css_for_each_child(tmp, last) + pos = tmp; + } while (pos); + + return last; +} + +static struct cgroup_subsys_state * +css_leftmost_descendant(struct cgroup_subsys_state *pos) +{ + struct cgroup_subsys_state *last; + + do { + last = pos; + pos = css_next_child(NULL, pos); + } while (pos); + + return last; +} + +/** + * css_next_descendant_post - find the next descendant for post-order walk + * @pos: the current position (%NULL to initiate traversal) + * @root: css whose descendants to walk + * + * To be used by css_for_each_descendant_post(). Find the next descendant + * to visit for post-order traversal of @root's descendants. @root is + * included in the iteration and the last node to be visited. + * + * While this function requires cgroup_mutex or RCU read locking, it + * doesn't require the whole traversal to be contained in a single critical + * section. This function will return the correct next descendant as long + * as both @pos and @cgroup are accessible and @pos is a descendant of + * @cgroup. + * + * If a subsystem synchronizes ->css_online() and the start of iteration, a + * css which finished ->css_online() is guaranteed to be visible in the + * future iterations and will stay visible until the last reference is put. + * A css which hasn't finished ->css_online() or already finished + * ->css_offline() may show up during traversal. It's each subsystem's + * responsibility to synchronize against on/offlining. + */ +struct cgroup_subsys_state * +css_next_descendant_post(struct cgroup_subsys_state *pos, + struct cgroup_subsys_state *root) +{ + struct cgroup_subsys_state *next; + + cgroup_assert_mutex_or_rcu_locked(); + + /* if first iteration, visit leftmost descendant which may be @root */ + if (!pos) + return css_leftmost_descendant(root); + + /* if we visited @root, we're done */ + if (pos == root) + return NULL; + + /* if there's an unvisited sibling, visit its leftmost descendant */ + next = css_next_child(pos, pos->parent); + if (next) + return css_leftmost_descendant(next); + + /* no sibling left, visit parent */ + return pos->parent; +} + +/** + * css_has_online_children - does a css have online children + * @css: the target css + * + * Returns %true if @css has any online children; otherwise, %false. This + * function can be called from any context but the caller is responsible + * for synchronizing against on/offlining as necessary. + */ +bool css_has_online_children(struct cgroup_subsys_state *css) +{ + struct cgroup_subsys_state *child; + bool ret = false; + + rcu_read_lock(); + css_for_each_child(child, css) { + if (child->flags & CSS_ONLINE) { + ret = true; + break; + } + } + rcu_read_unlock(); + return ret; +} + +/** + * css_task_iter_advance_css_set - advance a task itererator to the next css_set + * @it: the iterator to advance + * + * Advance @it to the next css_set to walk. + */ +static void css_task_iter_advance_css_set(struct css_task_iter *it) +{ + struct list_head *l = it->cset_pos; + struct cgrp_cset_link *link; + struct css_set *cset; + + lockdep_assert_held(&css_set_lock); + + /* Advance to the next non-empty css_set */ + do { + l = l->next; + if (l == it->cset_head) { + it->cset_pos = NULL; + it->task_pos = NULL; + return; + } + + if (it->ss) { + cset = container_of(l, struct css_set, + e_cset_node[it->ss->id]); + } else { + link = list_entry(l, struct cgrp_cset_link, cset_link); + cset = link->cset; + } + } while (!css_set_populated(cset)); + + it->cset_pos = l; + + if (!list_empty(&cset->tasks)) + it->task_pos = cset->tasks.next; + else + it->task_pos = cset->mg_tasks.next; + + it->tasks_head = &cset->tasks; + it->mg_tasks_head = &cset->mg_tasks; + + /* + * We don't keep css_sets locked across iteration steps and thus + * need to take steps to ensure that iteration can be resumed after + * the lock is re-acquired. Iteration is performed at two levels - + * css_sets and tasks in them. + * + * Once created, a css_set never leaves its cgroup lists, so a + * pinned css_set is guaranteed to stay put and we can resume + * iteration afterwards. + * + * Tasks may leave @cset across iteration steps. This is resolved + * by registering each iterator with the css_set currently being + * walked and making css_set_move_task() advance iterators whose + * next task is leaving. + */ + if (it->cur_cset) { + list_del(&it->iters_node); + put_css_set_locked(it->cur_cset); + } + get_css_set(cset); + it->cur_cset = cset; + list_add(&it->iters_node, &cset->task_iters); +} + +static void css_task_iter_advance(struct css_task_iter *it) +{ + struct list_head *l = it->task_pos; + + lockdep_assert_held(&css_set_lock); + WARN_ON_ONCE(!l); + + /* + * Advance iterator to find next entry. cset->tasks is consumed + * first and then ->mg_tasks. After ->mg_tasks, we move onto the + * next cset. + */ + l = l->next; + + if (l == it->tasks_head) + l = it->mg_tasks_head->next; + + if (l == it->mg_tasks_head) + css_task_iter_advance_css_set(it); + else + it->task_pos = l; +} + +/** + * css_task_iter_start - initiate task iteration + * @css: the css to walk tasks of + * @it: the task iterator to use + * + * Initiate iteration through the tasks of @css. The caller can call + * css_task_iter_next() to walk through the tasks until the function + * returns NULL. On completion of iteration, css_task_iter_end() must be + * called. + */ +void css_task_iter_start(struct cgroup_subsys_state *css, + struct css_task_iter *it) +{ + /* no one should try to iterate before mounting cgroups */ + WARN_ON_ONCE(!use_task_css_set_links); + + memset(it, 0, sizeof(*it)); + + spin_lock_irq(&css_set_lock); + + it->ss = css->ss; + + if (it->ss) + it->cset_pos = &css->cgroup->e_csets[css->ss->id]; + else + it->cset_pos = &css->cgroup->cset_links; + + it->cset_head = it->cset_pos; + + css_task_iter_advance_css_set(it); + + spin_unlock_irq(&css_set_lock); +} + +/** + * css_task_iter_next - return the next task for the iterator + * @it: the task iterator being iterated + * + * The "next" function for task iteration. @it should have been + * initialized via css_task_iter_start(). Returns NULL when the iteration + * reaches the end. + */ +struct task_struct *css_task_iter_next(struct css_task_iter *it) +{ + if (it->cur_task) { + put_task_struct(it->cur_task); + it->cur_task = NULL; + } + + spin_lock_irq(&css_set_lock); + + if (it->task_pos) { + it->cur_task = list_entry(it->task_pos, struct task_struct, + cg_list); + get_task_struct(it->cur_task); + css_task_iter_advance(it); + } + + spin_unlock_irq(&css_set_lock); + + return it->cur_task; +} + +/** + * css_task_iter_end - finish task iteration + * @it: the task iterator to finish + * + * Finish task iteration started by css_task_iter_start(). + */ +void css_task_iter_end(struct css_task_iter *it) +{ + if (it->cur_cset) { + spin_lock_irq(&css_set_lock); + list_del(&it->iters_node); + put_css_set_locked(it->cur_cset); + spin_unlock_irq(&css_set_lock); + } + + if (it->cur_task) + put_task_struct(it->cur_task); +} + +/** + * cgroup_trasnsfer_tasks - move tasks from one cgroup to another + * @to: cgroup to which the tasks will be moved + * @from: cgroup in which the tasks currently reside + * + * Locking rules between cgroup_post_fork() and the migration path + * guarantee that, if a task is forking while being migrated, the new child + * is guaranteed to be either visible in the source cgroup after the + * parent's migration is complete or put into the target cgroup. No task + * can slip out of migration through forking. + */ +int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from) +{ + LIST_HEAD(preloaded_csets); + struct cgrp_cset_link *link; + struct css_task_iter it; + struct task_struct *task; + int ret; + + if (cgroup_on_dfl(to)) + return -EINVAL; + + if (!cgroup_may_migrate_to(to)) + return -EBUSY; + + mutex_lock(&cgroup_mutex); + + percpu_down_write(&cgroup_threadgroup_rwsem); + + /* all tasks in @from are being moved, all csets are source */ + spin_lock_irq(&css_set_lock); + list_for_each_entry(link, &from->cset_links, cset_link) + cgroup_migrate_add_src(link->cset, to, &preloaded_csets); + spin_unlock_irq(&css_set_lock); + + ret = cgroup_migrate_prepare_dst(&preloaded_csets); + if (ret) + goto out_err; + + /* + * Migrate tasks one-by-one until @from is empty. This fails iff + * ->can_attach() fails. + */ + do { + css_task_iter_start(&from->self, &it); + task = css_task_iter_next(&it); + if (task) + get_task_struct(task); + css_task_iter_end(&it); + + if (task) { + ret = cgroup_migrate(task, false, to->root); + if (!ret) + trace_cgroup_transfer_tasks(to, task, false); + put_task_struct(task); + } + } while (task && !ret); +out_err: + cgroup_migrate_finish(&preloaded_csets); + percpu_up_write(&cgroup_threadgroup_rwsem); + mutex_unlock(&cgroup_mutex); + return ret; +} + +static void cgroup_procs_release(struct kernfs_open_file *of) +{ + if (of->priv) { + css_task_iter_end(of->priv); + kfree(of->priv); + } +} + +static void *cgroup_procs_next(struct seq_file *s, void *v, loff_t *pos) +{ + struct kernfs_open_file *of = s->private; + struct css_task_iter *it = of->priv; + struct task_struct *task; + + do { + task = css_task_iter_next(it); + } while (task && !thread_group_leader(task)); + + return task; +} + +static void *cgroup_procs_start(struct seq_file *s, loff_t *pos) +{ + struct kernfs_open_file *of = s->private; + struct cgroup *cgrp = seq_css(s)->cgroup; + struct css_task_iter *it = of->priv; + + /* + * When a seq_file is seeked, it's always traversed sequentially + * from position 0, so we can simply keep iterating on !0 *pos. + */ + if (!it) { + if (WARN_ON_ONCE((*pos)++)) + return ERR_PTR(-EINVAL); + + it = kzalloc(sizeof(*it), GFP_KERNEL); + if (!it) + return ERR_PTR(-ENOMEM); + of->priv = it; + css_task_iter_start(&cgrp->self, it); + } else if (!(*pos)++) { + css_task_iter_end(it); + css_task_iter_start(&cgrp->self, it); + } + + return cgroup_procs_next(s, NULL, NULL); +} + +static int cgroup_procs_show(struct seq_file *s, void *v) +{ + seq_printf(s, "%d\n", task_tgid_vnr(v)); + return 0; +} + +/* + * Stuff for reading the 'tasks'/'procs' files. + * + * Reading this file can return large amounts of data if a cgroup has + * *lots* of attached tasks. So it may need several calls to read(), + * but we cannot guarantee that the information we produce is correct + * unless we produce it entirely atomically. + * + */ + +/* which pidlist file are we talking about? */ +enum cgroup_filetype { + CGROUP_FILE_PROCS, + CGROUP_FILE_TASKS, +}; + +/* + * A pidlist is a list of pids that virtually represents the contents of one + * of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists, + * a pair (one each for procs, tasks) for each pid namespace that's relevant + * to the cgroup. + */ +struct cgroup_pidlist { + /* + * used to find which pidlist is wanted. doesn't change as long as + * this particular list stays in the list. + */ + struct { enum cgroup_filetype type; struct pid_namespace *ns; } key; + /* array of xids */ + pid_t *list; + /* how many elements the above list has */ + int length; + /* each of these stored in a list by its cgroup */ + struct list_head links; + /* pointer to the cgroup we belong to, for list removal purposes */ + struct cgroup *owner; + /* for delayed destruction */ + struct delayed_work destroy_dwork; +}; + +/* + * The following two functions "fix" the issue where there are more pids + * than kmalloc will give memory for; in such cases, we use vmalloc/vfree. + * TODO: replace with a kernel-wide solution to this problem + */ +#define PIDLIST_TOO_LARGE(c) ((c) * sizeof(pid_t) > (PAGE_SIZE * 2)) +static void *pidlist_allocate(int count) +{ + if (PIDLIST_TOO_LARGE(count)) + return vmalloc(count * sizeof(pid_t)); + else + return kmalloc(count * sizeof(pid_t), GFP_KERNEL); +} + +static void pidlist_free(void *p) +{ + kvfree(p); +} + +/* + * Used to destroy all pidlists lingering waiting for destroy timer. None + * should be left afterwards. + */ +static void cgroup_pidlist_destroy_all(struct cgroup *cgrp) +{ + struct cgroup_pidlist *l, *tmp_l; + + mutex_lock(&cgrp->pidlist_mutex); + list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links) + mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0); + mutex_unlock(&cgrp->pidlist_mutex); + + flush_workqueue(cgroup_pidlist_destroy_wq); + BUG_ON(!list_empty(&cgrp->pidlists)); +} + +static void cgroup_pidlist_destroy_work_fn(struct work_struct *work) +{ + struct delayed_work *dwork = to_delayed_work(work); + struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist, + destroy_dwork); + struct cgroup_pidlist *tofree = NULL; + + mutex_lock(&l->owner->pidlist_mutex); + + /* + * Destroy iff we didn't get queued again. The state won't change + * as destroy_dwork can only be queued while locked. + */ + if (!delayed_work_pending(dwork)) { + list_del(&l->links); + pidlist_free(l->list); + put_pid_ns(l->key.ns); + tofree = l; + } + + mutex_unlock(&l->owner->pidlist_mutex); + kfree(tofree); +} + +/* + * pidlist_uniq - given a kmalloc()ed list, strip out all duplicate entries + * Returns the number of unique elements. + */ +static int pidlist_uniq(pid_t *list, int length) +{ + int src, dest = 1; + + /* + * we presume the 0th element is unique, so i starts at 1. trivial + * edge cases first; no work needs to be done for either + */ + if (length == 0 || length == 1) + return length; + /* src and dest walk down the list; dest counts unique elements */ + for (src = 1; src < length; src++) { + /* find next unique element */ + while (list[src] == list[src-1]) { + src++; + if (src == length) + goto after; + } + /* dest always points to where the next unique element goes */ + list[dest] = list[src]; + dest++; + } +after: + return dest; +} + +/* + * The two pid files - task and cgroup.procs - guaranteed that the result + * is sorted, which forced this whole pidlist fiasco. As pid order is + * different per namespace, each namespace needs differently sorted list, + * making it impossible to use, for example, single rbtree of member tasks + * sorted by task pointer. As pidlists can be fairly large, allocating one + * per open file is dangerous, so cgroup had to implement shared pool of + * pidlists keyed by cgroup and namespace. + */ +static int cmppid(const void *a, const void *b) +{ + return *(pid_t *)a - *(pid_t *)b; +} + +static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp, + enum cgroup_filetype type) +{ + struct cgroup_pidlist *l; + /* don't need task_nsproxy() if we're looking at ourself */ + struct pid_namespace *ns = task_active_pid_ns(current); + + lockdep_assert_held(&cgrp->pidlist_mutex); + + list_for_each_entry(l, &cgrp->pidlists, links) + if (l->key.type == type && l->key.ns == ns) + return l; + return NULL; +} + +/* + * find the appropriate pidlist for our purpose (given procs vs tasks) + * returns with the lock on that pidlist already held, and takes care + * of the use count, or returns NULL with no locks held if we're out of + * memory. + */ +static struct cgroup_pidlist *cgroup_pidlist_find_create(struct cgroup *cgrp, + enum cgroup_filetype type) +{ + struct cgroup_pidlist *l; + + lockdep_assert_held(&cgrp->pidlist_mutex); + + l = cgroup_pidlist_find(cgrp, type); + if (l) + return l; + + /* entry not found; create a new one */ + l = kzalloc(sizeof(struct cgroup_pidlist), GFP_KERNEL); + if (!l) + return l; + + INIT_DELAYED_WORK(&l->destroy_dwork, cgroup_pidlist_destroy_work_fn); + l->key.type = type; + /* don't need task_nsproxy() if we're looking at ourself */ + l->key.ns = get_pid_ns(task_active_pid_ns(current)); + l->owner = cgrp; + list_add(&l->links, &cgrp->pidlists); + return l; +} + +/* + * Load a cgroup's pidarray with either procs' tgids or tasks' pids + */ +static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type, + struct cgroup_pidlist **lp) +{ + pid_t *array; + int length; + int pid, n = 0; /* used for populating the array */ + struct css_task_iter it; + struct task_struct *tsk; + struct cgroup_pidlist *l; + + lockdep_assert_held(&cgrp->pidlist_mutex); + + /* + * If cgroup gets more users after we read count, we won't have + * enough space - tough. This race is indistinguishable to the + * caller from the case that the additional cgroup users didn't + * show up until sometime later on. + */ + length = cgroup_task_count(cgrp); + array = pidlist_allocate(length); + if (!array) + return -ENOMEM; + /* now, populate the array */ + css_task_iter_start(&cgrp->self, &it); + while ((tsk = css_task_iter_next(&it))) { + if (unlikely(n == length)) + break; + /* get tgid or pid for procs or tasks file respectively */ + if (type == CGROUP_FILE_PROCS) + pid = task_tgid_vnr(tsk); + else + pid = task_pid_vnr(tsk); + if (pid > 0) /* make sure to only use valid results */ + array[n++] = pid; + } + css_task_iter_end(&it); + length = n; + /* now sort & (if procs) strip out duplicates */ + sort(array, length, sizeof(pid_t), cmppid, NULL); + if (type == CGROUP_FILE_PROCS) + length = pidlist_uniq(array, length); + + l = cgroup_pidlist_find_create(cgrp, type); + if (!l) { + pidlist_free(array); + return -ENOMEM; + } + + /* store array, freeing old if necessary */ + pidlist_free(l->list); + l->list = array; + l->length = length; + *lp = l; + return 0; +} + +/** + * cgroupstats_build - build and fill cgroupstats + * @stats: cgroupstats to fill information into + * @dentry: A dentry entry belonging to the cgroup for which stats have + * been requested. + * + * Build and fill cgroupstats so that taskstats can export it to user + * space. + */ +int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry) +{ + struct kernfs_node *kn = kernfs_node_from_dentry(dentry); + struct cgroup *cgrp; + struct css_task_iter it; + struct task_struct *tsk; + + /* it should be kernfs_node belonging to cgroupfs and is a directory */ + if (dentry->d_sb->s_type != &cgroup_fs_type || !kn || + kernfs_type(kn) != KERNFS_DIR) + return -EINVAL; + + mutex_lock(&cgroup_mutex); + + /* + * We aren't being called from kernfs and there's no guarantee on + * @kn->priv's validity. For this and css_tryget_online_from_dir(), + * @kn->priv is RCU safe. Let's do the RCU dancing. + */ + rcu_read_lock(); + cgrp = rcu_dereference(kn->priv); + if (!cgrp || cgroup_is_dead(cgrp)) { + rcu_read_unlock(); + mutex_unlock(&cgroup_mutex); + return -ENOENT; + } + rcu_read_unlock(); + + css_task_iter_start(&cgrp->self, &it); + while ((tsk = css_task_iter_next(&it))) { + switch (tsk->state) { + case TASK_RUNNING: + stats->nr_running++; + break; + case TASK_INTERRUPTIBLE: + stats->nr_sleeping++; + break; + case TASK_UNINTERRUPTIBLE: + stats->nr_uninterruptible++; + break; + case TASK_STOPPED: + stats->nr_stopped++; + break; + default: + if (delayacct_is_task_waiting_on_io(tsk)) + stats->nr_io_wait++; + break; + } + } + css_task_iter_end(&it); + + mutex_unlock(&cgroup_mutex); + return 0; +} + + +/* + * seq_file methods for the tasks/procs files. The seq_file position is the + * next pid to display; the seq_file iterator is a pointer to the pid + * in the cgroup->l->list array. + */ + +static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos) +{ + /* + * Initially we receive a position value that corresponds to + * one more than the last pid shown (or 0 on the first call or + * after a seek to the start). Use a binary-search to find the + * next pid to display, if any + */ + struct kernfs_open_file *of = s->private; + struct cgroup *cgrp = seq_css(s)->cgroup; + struct cgroup_pidlist *l; + enum cgroup_filetype type = seq_cft(s)->private; + int index = 0, pid = *pos; + int *iter, ret; + + mutex_lock(&cgrp->pidlist_mutex); + + /* + * !NULL @of->priv indicates that this isn't the first start() + * after open. If the matching pidlist is around, we can use that. + * Look for it. Note that @of->priv can't be used directly. It + * could already have been destroyed. + */ + if (of->priv) + of->priv = cgroup_pidlist_find(cgrp, type); + + /* + * Either this is the first start() after open or the matching + * pidlist has been destroyed inbetween. Create a new one. + */ + if (!of->priv) { + ret = pidlist_array_load(cgrp, type, + (struct cgroup_pidlist **)&of->priv); + if (ret) + return ERR_PTR(ret); + } + l = of->priv; + + if (pid) { + int end = l->length; + + while (index < end) { + int mid = (index + end) / 2; + if (l->list[mid] == pid) { + index = mid; + break; + } else if (l->list[mid] <= pid) + index = mid + 1; + else + end = mid; + } + } + /* If we're off the end of the array, we're done */ + if (index >= l->length) + return NULL; + /* Update the abstract position to be the actual pid that we found */ + iter = l->list + index; + *pos = *iter; + return iter; +} + +static void cgroup_pidlist_stop(struct seq_file *s, void *v) +{ + struct kernfs_open_file *of = s->private; + struct cgroup_pidlist *l = of->priv; + + if (l) + mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, + CGROUP_PIDLIST_DESTROY_DELAY); + mutex_unlock(&seq_css(s)->cgroup->pidlist_mutex); +} + +static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos) +{ + struct kernfs_open_file *of = s->private; + struct cgroup_pidlist *l = of->priv; + pid_t *p = v; + pid_t *end = l->list + l->length; + /* + * Advance to the next pid in the array. If this goes off the + * end, we're done + */ + p++; + if (p >= end) { + return NULL; + } else { + *pos = *p; + return p; + } +} + +static int cgroup_pidlist_show(struct seq_file *s, void *v) +{ + seq_printf(s, "%d\n", *(int *)v); + + return 0; +} + +static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + return notify_on_release(css->cgroup); +} + +static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css, + struct cftype *cft, u64 val) +{ + if (val) + set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags); + else + clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags); + return 0; +} + +static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); +} + +static int cgroup_clone_children_write(struct cgroup_subsys_state *css, + struct cftype *cft, u64 val) +{ + if (val) + set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); + else + clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); + return 0; +} + +/* cgroup core interface files for the default hierarchy */ +static struct cftype cgroup_dfl_base_files[] = { + { + .name = "cgroup.procs", + .file_offset = offsetof(struct cgroup, procs_file), + .release = cgroup_procs_release, + .seq_start = cgroup_procs_start, + .seq_next = cgroup_procs_next, + .seq_show = cgroup_procs_show, + .write = cgroup_procs_write, + }, + { + .name = "cgroup.controllers", + .seq_show = cgroup_controllers_show, + }, + { + .name = "cgroup.subtree_control", + .seq_show = cgroup_subtree_control_show, + .write = cgroup_subtree_control_write, + }, + { + .name = "cgroup.events", + .flags = CFTYPE_NOT_ON_ROOT, + .file_offset = offsetof(struct cgroup, events_file), + .seq_show = cgroup_events_show, + }, + { } /* terminate */ +}; + +/* cgroup core interface files for the legacy hierarchies */ +static struct cftype cgroup_legacy_base_files[] = { + { + .name = "cgroup.procs", + .seq_start = cgroup_pidlist_start, + .seq_next = cgroup_pidlist_next, + .seq_stop = cgroup_pidlist_stop, + .seq_show = cgroup_pidlist_show, + .private = CGROUP_FILE_PROCS, + .write = cgroup_procs_write, + }, + { + .name = "cgroup.clone_children", + .read_u64 = cgroup_clone_children_read, + .write_u64 = cgroup_clone_children_write, + }, + { + .name = "cgroup.sane_behavior", + .flags = CFTYPE_ONLY_ON_ROOT, + .seq_show = cgroup_sane_behavior_show, + }, + { + .name = "tasks", + .seq_start = cgroup_pidlist_start, + .seq_next = cgroup_pidlist_next, + .seq_stop = cgroup_pidlist_stop, + .seq_show = cgroup_pidlist_show, + .private = CGROUP_FILE_TASKS, + .write = cgroup_tasks_write, + }, + { + .name = "notify_on_release", + .read_u64 = cgroup_read_notify_on_release, + .write_u64 = cgroup_write_notify_on_release, + }, + { + .name = "release_agent", + .flags = CFTYPE_ONLY_ON_ROOT, + .seq_show = cgroup_release_agent_show, + .write = cgroup_release_agent_write, + .max_write_len = PATH_MAX - 1, + }, + { } /* terminate */ +}; + +/* + * css destruction is four-stage process. + * + * 1. Destruction starts. Killing of the percpu_ref is initiated. + * Implemented in kill_css(). + * + * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs + * and thus css_tryget_online() is guaranteed to fail, the css can be + * offlined by invoking offline_css(). After offlining, the base ref is + * put. Implemented in css_killed_work_fn(). + * + * 3. When the percpu_ref reaches zero, the only possible remaining + * accessors are inside RCU read sections. css_release() schedules the + * RCU callback. + * + * 4. After the grace period, the css can be freed. Implemented in + * css_free_work_fn(). + * + * It is actually hairier because both step 2 and 4 require process context + * and thus involve punting to css->destroy_work adding two additional + * steps to the already complex sequence. + */ +static void css_free_work_fn(struct work_struct *work) +{ + struct cgroup_subsys_state *css = + container_of(work, struct cgroup_subsys_state, destroy_work); + struct cgroup_subsys *ss = css->ss; + struct cgroup *cgrp = css->cgroup; + + percpu_ref_exit(&css->refcnt); + + if (ss) { + /* css free path */ + struct cgroup_subsys_state *parent = css->parent; + int id = css->id; + + ss->css_free(css); + cgroup_idr_remove(&ss->css_idr, id); + cgroup_put(cgrp); + + if (parent) + css_put(parent); + } else { + /* cgroup free path */ + atomic_dec(&cgrp->root->nr_cgrps); + cgroup_pidlist_destroy_all(cgrp); + cancel_work_sync(&cgrp->release_agent_work); + + if (cgroup_parent(cgrp)) { + /* + * We get a ref to the parent, and put the ref when + * this cgroup is being freed, so it's guaranteed + * that the parent won't be destroyed before its + * children. + */ + cgroup_put(cgroup_parent(cgrp)); + kernfs_put(cgrp->kn); + kfree(cgrp); + } else { + /* + * This is root cgroup's refcnt reaching zero, + * which indicates that the root should be + * released. + */ + cgroup_destroy_root(cgrp->root); + } + } +} + +static void css_free_rcu_fn(struct rcu_head *rcu_head) +{ + struct cgroup_subsys_state *css = + container_of(rcu_head, struct cgroup_subsys_state, rcu_head); + + INIT_WORK(&css->destroy_work, css_free_work_fn); + queue_work(cgroup_destroy_wq, &css->destroy_work); +} + +static void css_release_work_fn(struct work_struct *work) +{ + struct cgroup_subsys_state *css = + container_of(work, struct cgroup_subsys_state, destroy_work); + struct cgroup_subsys *ss = css->ss; + struct cgroup *cgrp = css->cgroup; + + mutex_lock(&cgroup_mutex); + + css->flags |= CSS_RELEASED; + list_del_rcu(&css->sibling); + + if (ss) { + /* css release path */ + cgroup_idr_replace(&ss->css_idr, NULL, css->id); + if (ss->css_released) + ss->css_released(css); + } else { + /* cgroup release path */ + trace_cgroup_release(cgrp); + + cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id); + cgrp->id = -1; + + /* + * There are two control paths which try to determine + * cgroup from dentry without going through kernfs - + * cgroupstats_build() and css_tryget_online_from_dir(). + * Those are supported by RCU protecting clearing of + * cgrp->kn->priv backpointer. + */ + if (cgrp->kn) + RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv, + NULL); + + cgroup_bpf_put(cgrp); + } + + mutex_unlock(&cgroup_mutex); + + call_rcu(&css->rcu_head, css_free_rcu_fn); +} + +static void css_release(struct percpu_ref *ref) +{ + struct cgroup_subsys_state *css = + container_of(ref, struct cgroup_subsys_state, refcnt); + + INIT_WORK(&css->destroy_work, css_release_work_fn); + queue_work(cgroup_destroy_wq, &css->destroy_work); +} + +static void init_and_link_css(struct cgroup_subsys_state *css, + struct cgroup_subsys *ss, struct cgroup *cgrp) +{ + lockdep_assert_held(&cgroup_mutex); + + cgroup_get(cgrp); + + memset(css, 0, sizeof(*css)); + css->cgroup = cgrp; + css->ss = ss; + css->id = -1; + INIT_LIST_HEAD(&css->sibling); + INIT_LIST_HEAD(&css->children); + css->serial_nr = css_serial_nr_next++; + atomic_set(&css->online_cnt, 0); + + if (cgroup_parent(cgrp)) { + css->parent = cgroup_css(cgroup_parent(cgrp), ss); + css_get(css->parent); + } + + BUG_ON(cgroup_css(cgrp, ss)); +} + +/* invoke ->css_online() on a new CSS and mark it online if successful */ +static int online_css(struct cgroup_subsys_state *css) +{ + struct cgroup_subsys *ss = css->ss; + int ret = 0; + + lockdep_assert_held(&cgroup_mutex); + + if (ss->css_online) + ret = ss->css_online(css); + if (!ret) { + css->flags |= CSS_ONLINE; + rcu_assign_pointer(css->cgroup->subsys[ss->id], css); + + atomic_inc(&css->online_cnt); + if (css->parent) + atomic_inc(&css->parent->online_cnt); + } + return ret; +} + +/* if the CSS is online, invoke ->css_offline() on it and mark it offline */ +static void offline_css(struct cgroup_subsys_state *css) +{ + struct cgroup_subsys *ss = css->ss; + + lockdep_assert_held(&cgroup_mutex); + + if (!(css->flags & CSS_ONLINE)) + return; + + if (ss->css_reset) + ss->css_reset(css); + + if (ss->css_offline) + ss->css_offline(css); + + css->flags &= ~CSS_ONLINE; + RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL); + + wake_up_all(&css->cgroup->offline_waitq); +} + +/** + * css_create - create a cgroup_subsys_state + * @cgrp: the cgroup new css will be associated with + * @ss: the subsys of new css + * + * Create a new css associated with @cgrp - @ss pair. On success, the new + * css is online and installed in @cgrp. This function doesn't create the + * interface files. Returns 0 on success, -errno on failure. + */ +static struct cgroup_subsys_state *css_create(struct cgroup *cgrp, + struct cgroup_subsys *ss) +{ + struct cgroup *parent = cgroup_parent(cgrp); + struct cgroup_subsys_state *parent_css = cgroup_css(parent, ss); + struct cgroup_subsys_state *css; + int err; + + lockdep_assert_held(&cgroup_mutex); + + css = ss->css_alloc(parent_css); + if (!css) + css = ERR_PTR(-ENOMEM); + if (IS_ERR(css)) + return css; + + init_and_link_css(css, ss, cgrp); + + err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL); + if (err) + goto err_free_css; + + err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_KERNEL); + if (err < 0) + goto err_free_css; + css->id = err; + + /* @css is ready to be brought online now, make it visible */ + list_add_tail_rcu(&css->sibling, &parent_css->children); + cgroup_idr_replace(&ss->css_idr, css, css->id); + + err = online_css(css); + if (err) + goto err_list_del; + + if (ss->broken_hierarchy && !ss->warned_broken_hierarchy && + cgroup_parent(parent)) { + pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n", + current->comm, current->pid, ss->name); + if (!strcmp(ss->name, "memory")) + pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n"); + ss->warned_broken_hierarchy = true; + } + + return css; + +err_list_del: + list_del_rcu(&css->sibling); +err_free_css: + call_rcu(&css->rcu_head, css_free_rcu_fn); + return ERR_PTR(err); +} + +static struct cgroup *cgroup_create(struct cgroup *parent) +{ + struct cgroup_root *root = parent->root; + struct cgroup *cgrp, *tcgrp; + int level = parent->level + 1; + int ret; + + /* allocate the cgroup and its ID, 0 is reserved for the root */ + cgrp = kzalloc(sizeof(*cgrp) + + sizeof(cgrp->ancestor_ids[0]) * (level + 1), GFP_KERNEL); + if (!cgrp) + return ERR_PTR(-ENOMEM); + + ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL); + if (ret) + goto out_free_cgrp; + + /* + * Temporarily set the pointer to NULL, so idr_find() won't return + * a half-baked cgroup. + */ + cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_KERNEL); + if (cgrp->id < 0) { + ret = -ENOMEM; + goto out_cancel_ref; + } + + init_cgroup_housekeeping(cgrp); + + cgrp->self.parent = &parent->self; + cgrp->root = root; + cgrp->level = level; + + for (tcgrp = cgrp; tcgrp; tcgrp = cgroup_parent(tcgrp)) + cgrp->ancestor_ids[tcgrp->level] = tcgrp->id; + + if (notify_on_release(parent)) + set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags); + + if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags)) + set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags); + + cgrp->self.serial_nr = css_serial_nr_next++; + + /* allocation complete, commit to creation */ + list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children); + atomic_inc(&root->nr_cgrps); + cgroup_get(parent); + + /* + * @cgrp is now fully operational. If something fails after this + * point, it'll be released via the normal destruction path. + */ + cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id); + + /* + * On the default hierarchy, a child doesn't automatically inherit + * subtree_control from the parent. Each is configured manually. + */ + if (!cgroup_on_dfl(cgrp)) + cgrp->subtree_control = cgroup_control(cgrp); + + if (parent) + cgroup_bpf_inherit(cgrp, parent); + + cgroup_propagate_control(cgrp); + + /* @cgrp doesn't have dir yet so the following will only create csses */ + ret = cgroup_apply_control_enable(cgrp); + if (ret) + goto out_destroy; + + return cgrp; + +out_cancel_ref: + percpu_ref_exit(&cgrp->self.refcnt); +out_free_cgrp: + kfree(cgrp); + return ERR_PTR(ret); +out_destroy: + cgroup_destroy_locked(cgrp); + return ERR_PTR(ret); +} + +static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, + umode_t mode) +{ + struct cgroup *parent, *cgrp; + struct kernfs_node *kn; + int ret; + + /* do not accept '\n' to prevent making /proc//cgroup unparsable */ + if (strchr(name, '\n')) + return -EINVAL; + + parent = cgroup_kn_lock_live(parent_kn, false); + if (!parent) + return -ENODEV; + + cgrp = cgroup_create(parent); + if (IS_ERR(cgrp)) { + ret = PTR_ERR(cgrp); + goto out_unlock; + } + + /* create the directory */ + kn = kernfs_create_dir(parent->kn, name, mode, cgrp); + if (IS_ERR(kn)) { + ret = PTR_ERR(kn); + goto out_destroy; + } + cgrp->kn = kn; + + /* + * This extra ref will be put in cgroup_free_fn() and guarantees + * that @cgrp->kn is always accessible. + */ + kernfs_get(kn); + + ret = cgroup_kn_set_ugid(kn); + if (ret) + goto out_destroy; + + ret = css_populate_dir(&cgrp->self); + if (ret) + goto out_destroy; + + ret = cgroup_apply_control_enable(cgrp); + if (ret) + goto out_destroy; + + trace_cgroup_mkdir(cgrp); + + /* let's create and online css's */ + kernfs_activate(kn); + + ret = 0; + goto out_unlock; + +out_destroy: + cgroup_destroy_locked(cgrp); +out_unlock: + cgroup_kn_unlock(parent_kn); + return ret; +} + +/* + * This is called when the refcnt of a css is confirmed to be killed. + * css_tryget_online() is now guaranteed to fail. Tell the subsystem to + * initate destruction and put the css ref from kill_css(). + */ +static void css_killed_work_fn(struct work_struct *work) +{ + struct cgroup_subsys_state *css = + container_of(work, struct cgroup_subsys_state, destroy_work); + + mutex_lock(&cgroup_mutex); + + do { + offline_css(css); + css_put(css); + /* @css can't go away while we're holding cgroup_mutex */ + css = css->parent; + } while (css && atomic_dec_and_test(&css->online_cnt)); + + mutex_unlock(&cgroup_mutex); +} + +/* css kill confirmation processing requires process context, bounce */ +static void css_killed_ref_fn(struct percpu_ref *ref) +{ + struct cgroup_subsys_state *css = + container_of(ref, struct cgroup_subsys_state, refcnt); + + if (atomic_dec_and_test(&css->online_cnt)) { + INIT_WORK(&css->destroy_work, css_killed_work_fn); + queue_work(cgroup_destroy_wq, &css->destroy_work); + } +} + +/** + * kill_css - destroy a css + * @css: css to destroy + * + * This function initiates destruction of @css by removing cgroup interface + * files and putting its base reference. ->css_offline() will be invoked + * asynchronously once css_tryget_online() is guaranteed to fail and when + * the reference count reaches zero, @css will be released. + */ +static void kill_css(struct cgroup_subsys_state *css) +{ + lockdep_assert_held(&cgroup_mutex); + + /* + * This must happen before css is disassociated with its cgroup. + * See seq_css() for details. + */ + css_clear_dir(css); + + /* + * Killing would put the base ref, but we need to keep it alive + * until after ->css_offline(). + */ + css_get(css); + + /* + * cgroup core guarantees that, by the time ->css_offline() is + * invoked, no new css reference will be given out via + * css_tryget_online(). We can't simply call percpu_ref_kill() and + * proceed to offlining css's because percpu_ref_kill() doesn't + * guarantee that the ref is seen as killed on all CPUs on return. + * + * Use percpu_ref_kill_and_confirm() to get notifications as each + * css is confirmed to be seen as killed on all CPUs. + */ + percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn); +} + +/** + * cgroup_destroy_locked - the first stage of cgroup destruction + * @cgrp: cgroup to be destroyed + * + * css's make use of percpu refcnts whose killing latency shouldn't be + * exposed to userland and are RCU protected. Also, cgroup core needs to + * guarantee that css_tryget_online() won't succeed by the time + * ->css_offline() is invoked. To satisfy all the requirements, + * destruction is implemented in the following two steps. + * + * s1. Verify @cgrp can be destroyed and mark it dying. Remove all + * userland visible parts and start killing the percpu refcnts of + * css's. Set up so that the next stage will be kicked off once all + * the percpu refcnts are confirmed to be killed. + * + * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the + * rest of destruction. Once all cgroup references are gone, the + * cgroup is RCU-freed. + * + * This function implements s1. After this step, @cgrp is gone as far as + * the userland is concerned and a new cgroup with the same name may be + * created. As cgroup doesn't care about the names internally, this + * doesn't cause any problem. + */ +static int cgroup_destroy_locked(struct cgroup *cgrp) + __releases(&cgroup_mutex) __acquires(&cgroup_mutex) +{ + struct cgroup_subsys_state *css; + struct cgrp_cset_link *link; + int ssid; + + lockdep_assert_held(&cgroup_mutex); + + /* + * Only migration can raise populated from zero and we're already + * holding cgroup_mutex. + */ + if (cgroup_is_populated(cgrp)) + return -EBUSY; + + /* + * Make sure there's no live children. We can't test emptiness of + * ->self.children as dead children linger on it while being + * drained; otherwise, "rmdir parent/child parent" may fail. + */ + if (css_has_online_children(&cgrp->self)) + return -EBUSY; + + /* + * Mark @cgrp and the associated csets dead. The former prevents + * further task migration and child creation by disabling + * cgroup_lock_live_group(). The latter makes the csets ignored by + * the migration path. + */ + cgrp->self.flags &= ~CSS_ONLINE; + + spin_lock_irq(&css_set_lock); + list_for_each_entry(link, &cgrp->cset_links, cset_link) + link->cset->dead = true; + spin_unlock_irq(&css_set_lock); + + /* initiate massacre of all css's */ + for_each_css(css, ssid, cgrp) + kill_css(css); + + /* + * Remove @cgrp directory along with the base files. @cgrp has an + * extra ref on its kn. + */ + kernfs_remove(cgrp->kn); + + check_for_release(cgroup_parent(cgrp)); + + /* put the base reference */ + percpu_ref_kill(&cgrp->self.refcnt); + + return 0; +}; + +static int cgroup_rmdir(struct kernfs_node *kn) +{ + struct cgroup *cgrp; + int ret = 0; + + cgrp = cgroup_kn_lock_live(kn, false); + if (!cgrp) + return 0; + + ret = cgroup_destroy_locked(cgrp); + + if (!ret) + trace_cgroup_rmdir(cgrp); + + cgroup_kn_unlock(kn); + return ret; +} + +static struct kernfs_syscall_ops cgroup_kf_syscall_ops = { + .remount_fs = cgroup_remount, + .show_options = cgroup_show_options, + .mkdir = cgroup_mkdir, + .rmdir = cgroup_rmdir, + .rename = cgroup_rename, + .show_path = cgroup_show_path, +}; + +static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early) +{ + struct cgroup_subsys_state *css; + + pr_debug("Initializing cgroup subsys %s\n", ss->name); + + mutex_lock(&cgroup_mutex); + + idr_init(&ss->css_idr); + INIT_LIST_HEAD(&ss->cfts); + + /* Create the root cgroup state for this subsystem */ + ss->root = &cgrp_dfl_root; + css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss)); + /* We don't handle early failures gracefully */ + BUG_ON(IS_ERR(css)); + init_and_link_css(css, ss, &cgrp_dfl_root.cgrp); + + /* + * Root csses are never destroyed and we can't initialize + * percpu_ref during early init. Disable refcnting. + */ + css->flags |= CSS_NO_REF; + + if (early) { + /* allocation can't be done safely during early init */ + css->id = 1; + } else { + css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL); + BUG_ON(css->id < 0); + } + + /* Update the init_css_set to contain a subsys + * pointer to this state - since the subsystem is + * newly registered, all tasks and hence the + * init_css_set is in the subsystem's root cgroup. */ + init_css_set.subsys[ss->id] = css; + + have_fork_callback |= (bool)ss->fork << ss->id; + have_exit_callback |= (bool)ss->exit << ss->id; + have_free_callback |= (bool)ss->free << ss->id; + have_canfork_callback |= (bool)ss->can_fork << ss->id; + + /* At system boot, before all subsystems have been + * registered, no tasks have been forked, so we don't + * need to invoke fork callbacks here. */ + BUG_ON(!list_empty(&init_task.tasks)); + + BUG_ON(online_css(css)); + + mutex_unlock(&cgroup_mutex); +} + +/** + * cgroup_init_early - cgroup initialization at system boot + * + * Initialize cgroups at system boot, and initialize any + * subsystems that request early init. + */ +int __init cgroup_init_early(void) +{ + static struct cgroup_sb_opts __initdata opts; + struct cgroup_subsys *ss; + int i; + + init_cgroup_root(&cgrp_dfl_root, &opts); + cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF; + + RCU_INIT_POINTER(init_task.cgroups, &init_css_set); + + for_each_subsys(ss, i) { + WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id, + "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p id:name=%d:%s\n", + i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free, + ss->id, ss->name); + WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN, + "cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]); + + ss->id = i; + ss->name = cgroup_subsys_name[i]; + if (!ss->legacy_name) + ss->legacy_name = cgroup_subsys_name[i]; + + if (ss->early_init) + cgroup_init_subsys(ss, true); + } + return 0; +} + +static u16 cgroup_disable_mask __initdata; + +/** + * cgroup_init - cgroup initialization + * + * Register cgroup filesystem and /proc file, and initialize + * any subsystems that didn't request early init. + */ +int __init cgroup_init(void) +{ + struct cgroup_subsys *ss; + int ssid; + + BUILD_BUG_ON(CGROUP_SUBSYS_COUNT > 16); + BUG_ON(percpu_init_rwsem(&cgroup_threadgroup_rwsem)); + BUG_ON(cgroup_init_cftypes(NULL, cgroup_dfl_base_files)); + BUG_ON(cgroup_init_cftypes(NULL, cgroup_legacy_base_files)); + + /* + * The latency of the synchronize_sched() is too high for cgroups, + * avoid it at the cost of forcing all readers into the slow path. + */ + rcu_sync_enter_start(&cgroup_threadgroup_rwsem.rss); + + get_user_ns(init_cgroup_ns.user_ns); + + mutex_lock(&cgroup_mutex); + + /* + * Add init_css_set to the hash table so that dfl_root can link to + * it during init. + */ + hash_add(css_set_table, &init_css_set.hlist, + css_set_hash(init_css_set.subsys)); + + BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0)); + + mutex_unlock(&cgroup_mutex); + + for_each_subsys(ss, ssid) { + if (ss->early_init) { + struct cgroup_subsys_state *css = + init_css_set.subsys[ss->id]; + + css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, + GFP_KERNEL); + BUG_ON(css->id < 0); + } else { + cgroup_init_subsys(ss, false); + } + + list_add_tail(&init_css_set.e_cset_node[ssid], + &cgrp_dfl_root.cgrp.e_csets[ssid]); + + /* + * Setting dfl_root subsys_mask needs to consider the + * disabled flag and cftype registration needs kmalloc, + * both of which aren't available during early_init. + */ + if (cgroup_disable_mask & (1 << ssid)) { + static_branch_disable(cgroup_subsys_enabled_key[ssid]); + printk(KERN_INFO "Disabling %s control group subsystem\n", + ss->name); + continue; + } + + if (cgroup_ssid_no_v1(ssid)) + printk(KERN_INFO "Disabling %s control group subsystem in v1 mounts\n", + ss->name); + + cgrp_dfl_root.subsys_mask |= 1 << ss->id; + + if (ss->implicit_on_dfl) + cgrp_dfl_implicit_ss_mask |= 1 << ss->id; + else if (!ss->dfl_cftypes) + cgrp_dfl_inhibit_ss_mask |= 1 << ss->id; + + if (ss->dfl_cftypes == ss->legacy_cftypes) { + WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes)); + } else { + WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes)); + WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes)); + } + + if (ss->bind) + ss->bind(init_css_set.subsys[ssid]); + } + + /* init_css_set.subsys[] has been updated, re-hash */ + hash_del(&init_css_set.hlist); + hash_add(css_set_table, &init_css_set.hlist, + css_set_hash(init_css_set.subsys)); + + WARN_ON(sysfs_create_mount_point(fs_kobj, "cgroup")); + WARN_ON(register_filesystem(&cgroup_fs_type)); + WARN_ON(register_filesystem(&cgroup2_fs_type)); + WARN_ON(!proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations)); + + return 0; +} + +static int __init cgroup_wq_init(void) +{ + /* + * There isn't much point in executing destruction path in + * parallel. Good chunk is serialized with cgroup_mutex anyway. + * Use 1 for @max_active. + * + * We would prefer to do this in cgroup_init() above, but that + * is called before init_workqueues(): so leave this until after. + */ + cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1); + BUG_ON(!cgroup_destroy_wq); + + /* + * Used to destroy pidlists and separate to serve as flush domain. + * Cap @max_active to 1 too. + */ + cgroup_pidlist_destroy_wq = alloc_workqueue("cgroup_pidlist_destroy", + 0, 1); + BUG_ON(!cgroup_pidlist_destroy_wq); + + return 0; +} +core_initcall(cgroup_wq_init); + +/* + * proc_cgroup_show() + * - Print task's cgroup paths into seq_file, one line for each hierarchy + * - Used for /proc//cgroup. + */ +int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *tsk) +{ + char *buf; + int retval; + struct cgroup_root *root; + + retval = -ENOMEM; + buf = kmalloc(PATH_MAX, GFP_KERNEL); + if (!buf) + goto out; + + mutex_lock(&cgroup_mutex); + spin_lock_irq(&css_set_lock); + + for_each_root(root) { + struct cgroup_subsys *ss; + struct cgroup *cgrp; + int ssid, count = 0; + + if (root == &cgrp_dfl_root && !cgrp_dfl_visible) + continue; + + seq_printf(m, "%d:", root->hierarchy_id); + if (root != &cgrp_dfl_root) + for_each_subsys(ss, ssid) + if (root->subsys_mask & (1 << ssid)) + seq_printf(m, "%s%s", count++ ? "," : "", + ss->legacy_name); + if (strlen(root->name)) + seq_printf(m, "%sname=%s", count ? "," : "", + root->name); + seq_putc(m, ':'); + + cgrp = task_cgroup_from_root(tsk, root); + + /* + * On traditional hierarchies, all zombie tasks show up as + * belonging to the root cgroup. On the default hierarchy, + * while a zombie doesn't show up in "cgroup.procs" and + * thus can't be migrated, its /proc/PID/cgroup keeps + * reporting the cgroup it belonged to before exiting. If + * the cgroup is removed before the zombie is reaped, + * " (deleted)" is appended to the cgroup path. + */ + if (cgroup_on_dfl(cgrp) || !(tsk->flags & PF_EXITING)) { + retval = cgroup_path_ns_locked(cgrp, buf, PATH_MAX, + current->nsproxy->cgroup_ns); + if (retval >= PATH_MAX) + retval = -ENAMETOOLONG; + if (retval < 0) + goto out_unlock; + + seq_puts(m, buf); + } else { + seq_puts(m, "/"); + } + + if (cgroup_on_dfl(cgrp) && cgroup_is_dead(cgrp)) + seq_puts(m, " (deleted)\n"); + else + seq_putc(m, '\n'); + } + + retval = 0; +out_unlock: + spin_unlock_irq(&css_set_lock); + mutex_unlock(&cgroup_mutex); + kfree(buf); +out: + return retval; +} + +/* Display information about each subsystem and each hierarchy */ +static int proc_cgroupstats_show(struct seq_file *m, void *v) +{ + struct cgroup_subsys *ss; + int i; + + seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n"); + /* + * ideally we don't want subsystems moving around while we do this. + * cgroup_mutex is also necessary to guarantee an atomic snapshot of + * subsys/hierarchy state. + */ + mutex_lock(&cgroup_mutex); + + for_each_subsys(ss, i) + seq_printf(m, "%s\t%d\t%d\t%d\n", + ss->legacy_name, ss->root->hierarchy_id, + atomic_read(&ss->root->nr_cgrps), + cgroup_ssid_enabled(i)); + + mutex_unlock(&cgroup_mutex); + return 0; +} + +static int cgroupstats_open(struct inode *inode, struct file *file) +{ + return single_open(file, proc_cgroupstats_show, NULL); +} + +static const struct file_operations proc_cgroupstats_operations = { + .open = cgroupstats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/** + * cgroup_fork - initialize cgroup related fields during copy_process() + * @child: pointer to task_struct of forking parent process. + * + * A task is associated with the init_css_set until cgroup_post_fork() + * attaches it to the parent's css_set. Empty cg_list indicates that + * @child isn't holding reference to its css_set. + */ +void cgroup_fork(struct task_struct *child) +{ + RCU_INIT_POINTER(child->cgroups, &init_css_set); + INIT_LIST_HEAD(&child->cg_list); +} + +/** + * cgroup_can_fork - called on a new task before the process is exposed + * @child: the task in question. + * + * This calls the subsystem can_fork() callbacks. If the can_fork() callback + * returns an error, the fork aborts with that error code. This allows for + * a cgroup subsystem to conditionally allow or deny new forks. + */ +int cgroup_can_fork(struct task_struct *child) +{ + struct cgroup_subsys *ss; + int i, j, ret; + + do_each_subsys_mask(ss, i, have_canfork_callback) { + ret = ss->can_fork(child); + if (ret) + goto out_revert; + } while_each_subsys_mask(); + + return 0; + +out_revert: + for_each_subsys(ss, j) { + if (j >= i) + break; + if (ss->cancel_fork) + ss->cancel_fork(child); + } + + return ret; +} + +/** + * cgroup_cancel_fork - called if a fork failed after cgroup_can_fork() + * @child: the task in question + * + * This calls the cancel_fork() callbacks if a fork failed *after* + * cgroup_can_fork() succeded. + */ +void cgroup_cancel_fork(struct task_struct *child) +{ + struct cgroup_subsys *ss; + int i; + + for_each_subsys(ss, i) + if (ss->cancel_fork) + ss->cancel_fork(child); +} + +/** + * cgroup_post_fork - called on a new task after adding it to the task list + * @child: the task in question + * + * Adds the task to the list running through its css_set if necessary and + * call the subsystem fork() callbacks. Has to be after the task is + * visible on the task list in case we race with the first call to + * cgroup_task_iter_start() - to guarantee that the new task ends up on its + * list. + */ +void cgroup_post_fork(struct task_struct *child) +{ + struct cgroup_subsys *ss; + int i; + + /* + * This may race against cgroup_enable_task_cg_lists(). As that + * function sets use_task_css_set_links before grabbing + * tasklist_lock and we just went through tasklist_lock to add + * @child, it's guaranteed that either we see the set + * use_task_css_set_links or cgroup_enable_task_cg_lists() sees + * @child during its iteration. + * + * If we won the race, @child is associated with %current's + * css_set. Grabbing css_set_lock guarantees both that the + * association is stable, and, on completion of the parent's + * migration, @child is visible in the source of migration or + * already in the destination cgroup. This guarantee is necessary + * when implementing operations which need to migrate all tasks of + * a cgroup to another. + * + * Note that if we lose to cgroup_enable_task_cg_lists(), @child + * will remain in init_css_set. This is safe because all tasks are + * in the init_css_set before cg_links is enabled and there's no + * operation which transfers all tasks out of init_css_set. + */ + if (use_task_css_set_links) { + struct css_set *cset; + + spin_lock_irq(&css_set_lock); + cset = task_css_set(current); + if (list_empty(&child->cg_list)) { + get_css_set(cset); + css_set_move_task(child, NULL, cset, false); + } + spin_unlock_irq(&css_set_lock); + } + + /* + * Call ss->fork(). This must happen after @child is linked on + * css_set; otherwise, @child might change state between ->fork() + * and addition to css_set. + */ + do_each_subsys_mask(ss, i, have_fork_callback) { + ss->fork(child); + } while_each_subsys_mask(); +} + +/** + * cgroup_exit - detach cgroup from exiting task + * @tsk: pointer to task_struct of exiting process + * + * Description: Detach cgroup from @tsk and release it. + * + * Note that cgroups marked notify_on_release force every task in + * them to take the global cgroup_mutex mutex when exiting. + * This could impact scaling on very large systems. Be reluctant to + * use notify_on_release cgroups where very high task exit scaling + * is required on large systems. + * + * We set the exiting tasks cgroup to the root cgroup (top_cgroup). We + * call cgroup_exit() while the task is still competent to handle + * notify_on_release(), then leave the task attached to the root cgroup in + * each hierarchy for the remainder of its exit. No need to bother with + * init_css_set refcnting. init_css_set never goes away and we can't race + * with migration path - PF_EXITING is visible to migration path. + */ +void cgroup_exit(struct task_struct *tsk) +{ + struct cgroup_subsys *ss; + struct css_set *cset; + int i; + + /* + * Unlink from @tsk from its css_set. As migration path can't race + * with us, we can check css_set and cg_list without synchronization. + */ + cset = task_css_set(tsk); + + if (!list_empty(&tsk->cg_list)) { + spin_lock_irq(&css_set_lock); + css_set_move_task(tsk, cset, NULL, false); + spin_unlock_irq(&css_set_lock); + } else { + get_css_set(cset); + } + + /* see cgroup_post_fork() for details */ + do_each_subsys_mask(ss, i, have_exit_callback) { + ss->exit(tsk); + } while_each_subsys_mask(); +} + +void cgroup_free(struct task_struct *task) +{ + struct css_set *cset = task_css_set(task); + struct cgroup_subsys *ss; + int ssid; + + do_each_subsys_mask(ss, ssid, have_free_callback) { + ss->free(task); + } while_each_subsys_mask(); + + put_css_set(cset); +} + +static void check_for_release(struct cgroup *cgrp) +{ + if (notify_on_release(cgrp) && !cgroup_is_populated(cgrp) && + !css_has_online_children(&cgrp->self) && !cgroup_is_dead(cgrp)) + schedule_work(&cgrp->release_agent_work); +} + +/* + * Notify userspace when a cgroup is released, by running the + * configured release agent with the name of the cgroup (path + * relative to the root of cgroup file system) as the argument. + * + * Most likely, this user command will try to rmdir this cgroup. + * + * This races with the possibility that some other task will be + * attached to this cgroup before it is removed, or that some other + * user task will 'mkdir' a child cgroup of this cgroup. That's ok. + * The presumed 'rmdir' will fail quietly if this cgroup is no longer + * unused, and this cgroup will be reprieved from its death sentence, + * to continue to serve a useful existence. Next time it's released, + * we will get notified again, if it still has 'notify_on_release' set. + * + * The final arg to call_usermodehelper() is UMH_WAIT_EXEC, which + * means only wait until the task is successfully execve()'d. The + * separate release agent task is forked by call_usermodehelper(), + * then control in this thread returns here, without waiting for the + * release agent task. We don't bother to wait because the caller of + * this routine has no use for the exit status of the release agent + * task, so no sense holding our caller up for that. + */ +static void cgroup_release_agent(struct work_struct *work) +{ + struct cgroup *cgrp = + container_of(work, struct cgroup, release_agent_work); + char *pathbuf = NULL, *agentbuf = NULL; + char *argv[3], *envp[3]; + int ret; + + mutex_lock(&cgroup_mutex); + + pathbuf = kmalloc(PATH_MAX, GFP_KERNEL); + agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL); + if (!pathbuf || !agentbuf) + goto out; + + spin_lock_irq(&css_set_lock); + ret = cgroup_path_ns_locked(cgrp, pathbuf, PATH_MAX, &init_cgroup_ns); + spin_unlock_irq(&css_set_lock); + if (ret < 0 || ret >= PATH_MAX) + goto out; + + argv[0] = agentbuf; + argv[1] = pathbuf; + argv[2] = NULL; + + /* minimal command environment */ + envp[0] = "HOME=/"; + envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin"; + envp[2] = NULL; + + mutex_unlock(&cgroup_mutex); + call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC); + goto out_free; +out: + mutex_unlock(&cgroup_mutex); +out_free: + kfree(agentbuf); + kfree(pathbuf); +} + +static int __init cgroup_disable(char *str) +{ + struct cgroup_subsys *ss; + char *token; + int i; + + while ((token = strsep(&str, ",")) != NULL) { + if (!*token) + continue; + + for_each_subsys(ss, i) { + if (strcmp(token, ss->name) && + strcmp(token, ss->legacy_name)) + continue; + cgroup_disable_mask |= 1 << i; + } + } + return 1; +} +__setup("cgroup_disable=", cgroup_disable); + +static int __init cgroup_no_v1(char *str) +{ + struct cgroup_subsys *ss; + char *token; + int i; + + while ((token = strsep(&str, ",")) != NULL) { + if (!*token) + continue; + + if (!strcmp(token, "all")) { + cgroup_no_v1_mask = U16_MAX; + break; + } + + for_each_subsys(ss, i) { + if (strcmp(token, ss->name) && + strcmp(token, ss->legacy_name)) + continue; + + cgroup_no_v1_mask |= 1 << i; + } + } + return 1; +} +__setup("cgroup_no_v1=", cgroup_no_v1); + +/** + * css_tryget_online_from_dir - get corresponding css from a cgroup dentry + * @dentry: directory dentry of interest + * @ss: subsystem of interest + * + * If @dentry is a directory for a cgroup which has @ss enabled on it, try + * to get the corresponding css and return it. If such css doesn't exist + * or can't be pinned, an ERR_PTR value is returned. + */ +struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry, + struct cgroup_subsys *ss) +{ + struct kernfs_node *kn = kernfs_node_from_dentry(dentry); + struct file_system_type *s_type = dentry->d_sb->s_type; + struct cgroup_subsys_state *css = NULL; + struct cgroup *cgrp; + + /* is @dentry a cgroup dir? */ + if ((s_type != &cgroup_fs_type && s_type != &cgroup2_fs_type) || + !kn || kernfs_type(kn) != KERNFS_DIR) + return ERR_PTR(-EBADF); + + rcu_read_lock(); + + /* + * This path doesn't originate from kernfs and @kn could already + * have been or be removed at any point. @kn->priv is RCU + * protected for this access. See css_release_work_fn() for details. + */ + cgrp = rcu_dereference(kn->priv); + if (cgrp) + css = cgroup_css(cgrp, ss); + + if (!css || !css_tryget_online(css)) + css = ERR_PTR(-ENOENT); + + rcu_read_unlock(); + return css; +} + +/** + * css_from_id - lookup css by id + * @id: the cgroup id + * @ss: cgroup subsys to be looked into + * + * Returns the css if there's valid one with @id, otherwise returns NULL. + * Should be called under rcu_read_lock(). + */ +struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss) +{ + WARN_ON_ONCE(!rcu_read_lock_held()); + return idr_find(&ss->css_idr, id); +} + +/** + * cgroup_get_from_path - lookup and get a cgroup from its default hierarchy path + * @path: path on the default hierarchy + * + * Find the cgroup at @path on the default hierarchy, increment its + * reference count and return it. Returns pointer to the found cgroup on + * success, ERR_PTR(-ENOENT) if @path doens't exist and ERR_PTR(-ENOTDIR) + * if @path points to a non-directory. + */ +struct cgroup *cgroup_get_from_path(const char *path) +{ + struct kernfs_node *kn; + struct cgroup *cgrp; + + mutex_lock(&cgroup_mutex); + + kn = kernfs_walk_and_get(cgrp_dfl_root.cgrp.kn, path); + if (kn) { + if (kernfs_type(kn) == KERNFS_DIR) { + cgrp = kn->priv; + cgroup_get(cgrp); + } else { + cgrp = ERR_PTR(-ENOTDIR); + } + kernfs_put(kn); + } else { + cgrp = ERR_PTR(-ENOENT); + } + + mutex_unlock(&cgroup_mutex); + return cgrp; +} +EXPORT_SYMBOL_GPL(cgroup_get_from_path); + +/** + * cgroup_get_from_fd - get a cgroup pointer from a fd + * @fd: fd obtained by open(cgroup2_dir) + * + * Find the cgroup from a fd which should be obtained + * by opening a cgroup directory. Returns a pointer to the + * cgroup on success. ERR_PTR is returned if the cgroup + * cannot be found. + */ +struct cgroup *cgroup_get_from_fd(int fd) +{ + struct cgroup_subsys_state *css; + struct cgroup *cgrp; + struct file *f; + + f = fget_raw(fd); + if (!f) + return ERR_PTR(-EBADF); + + css = css_tryget_online_from_dir(f->f_path.dentry, NULL); + fput(f); + if (IS_ERR(css)) + return ERR_CAST(css); + + cgrp = css->cgroup; + if (!cgroup_on_dfl(cgrp)) { + cgroup_put(cgrp); + return ERR_PTR(-EBADF); + } + + return cgrp; +} +EXPORT_SYMBOL_GPL(cgroup_get_from_fd); + +/* + * sock->sk_cgrp_data handling. For more info, see sock_cgroup_data + * definition in cgroup-defs.h. + */ +#ifdef CONFIG_SOCK_CGROUP_DATA + +#if defined(CONFIG_CGROUP_NET_PRIO) || defined(CONFIG_CGROUP_NET_CLASSID) + +DEFINE_SPINLOCK(cgroup_sk_update_lock); +static bool cgroup_sk_alloc_disabled __read_mostly; + +void cgroup_sk_alloc_disable(void) +{ + if (cgroup_sk_alloc_disabled) + return; + pr_info("cgroup: disabling cgroup2 socket matching due to net_prio or net_cls activation\n"); + cgroup_sk_alloc_disabled = true; +} + +#else + +#define cgroup_sk_alloc_disabled false + +#endif + +void cgroup_sk_alloc(struct sock_cgroup_data *skcd) +{ + if (cgroup_sk_alloc_disabled) + return; + + /* Socket clone path */ + if (skcd->val) { + cgroup_get(sock_cgroup_ptr(skcd)); + return; + } + + rcu_read_lock(); + + while (true) { + struct css_set *cset; + + cset = task_css_set(current); + if (likely(cgroup_tryget(cset->dfl_cgrp))) { + skcd->val = (unsigned long)cset->dfl_cgrp; + break; + } + cpu_relax(); + } + + rcu_read_unlock(); +} + +void cgroup_sk_free(struct sock_cgroup_data *skcd) +{ + cgroup_put(sock_cgroup_ptr(skcd)); +} + +#endif /* CONFIG_SOCK_CGROUP_DATA */ + +/* cgroup namespaces */ + +static struct ucounts *inc_cgroup_namespaces(struct user_namespace *ns) +{ + return inc_ucount(ns, current_euid(), UCOUNT_CGROUP_NAMESPACES); +} + +static void dec_cgroup_namespaces(struct ucounts *ucounts) +{ + dec_ucount(ucounts, UCOUNT_CGROUP_NAMESPACES); +} + +static struct cgroup_namespace *alloc_cgroup_ns(void) +{ + struct cgroup_namespace *new_ns; + int ret; + + new_ns = kzalloc(sizeof(struct cgroup_namespace), GFP_KERNEL); + if (!new_ns) + return ERR_PTR(-ENOMEM); + ret = ns_alloc_inum(&new_ns->ns); + if (ret) { + kfree(new_ns); + return ERR_PTR(ret); + } + atomic_set(&new_ns->count, 1); + new_ns->ns.ops = &cgroupns_operations; + return new_ns; +} + +void free_cgroup_ns(struct cgroup_namespace *ns) +{ + put_css_set(ns->root_cset); + dec_cgroup_namespaces(ns->ucounts); + put_user_ns(ns->user_ns); + ns_free_inum(&ns->ns); + kfree(ns); +} +EXPORT_SYMBOL(free_cgroup_ns); + +struct cgroup_namespace *copy_cgroup_ns(unsigned long flags, + struct user_namespace *user_ns, + struct cgroup_namespace *old_ns) +{ + struct cgroup_namespace *new_ns; + struct ucounts *ucounts; + struct css_set *cset; + + BUG_ON(!old_ns); + + if (!(flags & CLONE_NEWCGROUP)) { + get_cgroup_ns(old_ns); + return old_ns; + } + + /* Allow only sysadmin to create cgroup namespace. */ + if (!ns_capable(user_ns, CAP_SYS_ADMIN)) + return ERR_PTR(-EPERM); + + ucounts = inc_cgroup_namespaces(user_ns); + if (!ucounts) + return ERR_PTR(-ENOSPC); + + /* It is not safe to take cgroup_mutex here */ + spin_lock_irq(&css_set_lock); + cset = task_css_set(current); + get_css_set(cset); + spin_unlock_irq(&css_set_lock); + + new_ns = alloc_cgroup_ns(); + if (IS_ERR(new_ns)) { + put_css_set(cset); + dec_cgroup_namespaces(ucounts); + return new_ns; + } + + new_ns->user_ns = get_user_ns(user_ns); + new_ns->ucounts = ucounts; + new_ns->root_cset = cset; + + return new_ns; +} + +static inline struct cgroup_namespace *to_cg_ns(struct ns_common *ns) +{ + return container_of(ns, struct cgroup_namespace, ns); +} + +static int cgroupns_install(struct nsproxy *nsproxy, struct ns_common *ns) +{ + struct cgroup_namespace *cgroup_ns = to_cg_ns(ns); + + if (!ns_capable(current_user_ns(), CAP_SYS_ADMIN) || + !ns_capable(cgroup_ns->user_ns, CAP_SYS_ADMIN)) + return -EPERM; + + /* Don't need to do anything if we are attaching to our own cgroupns. */ + if (cgroup_ns == nsproxy->cgroup_ns) + return 0; + + get_cgroup_ns(cgroup_ns); + put_cgroup_ns(nsproxy->cgroup_ns); + nsproxy->cgroup_ns = cgroup_ns; + + return 0; +} + +static struct ns_common *cgroupns_get(struct task_struct *task) +{ + struct cgroup_namespace *ns = NULL; + struct nsproxy *nsproxy; + + task_lock(task); + nsproxy = task->nsproxy; + if (nsproxy) { + ns = nsproxy->cgroup_ns; + get_cgroup_ns(ns); + } + task_unlock(task); + + return ns ? &ns->ns : NULL; +} + +static void cgroupns_put(struct ns_common *ns) +{ + put_cgroup_ns(to_cg_ns(ns)); +} + +static struct user_namespace *cgroupns_owner(struct ns_common *ns) +{ + return to_cg_ns(ns)->user_ns; +} + +const struct proc_ns_operations cgroupns_operations = { + .name = "cgroup", + .type = CLONE_NEWCGROUP, + .get = cgroupns_get, + .put = cgroupns_put, + .install = cgroupns_install, + .owner = cgroupns_owner, +}; + +static __init int cgroup_namespaces_init(void) +{ + return 0; +} +subsys_initcall(cgroup_namespaces_init); + +#ifdef CONFIG_CGROUP_BPF +void cgroup_bpf_update(struct cgroup *cgrp, + struct bpf_prog *prog, + enum bpf_attach_type type) +{ + struct cgroup *parent = cgroup_parent(cgrp); + + mutex_lock(&cgroup_mutex); + __cgroup_bpf_update(cgrp, parent, prog, type); + mutex_unlock(&cgroup_mutex); +} +#endif /* CONFIG_CGROUP_BPF */ + +#ifdef CONFIG_CGROUP_DEBUG +static struct cgroup_subsys_state * +debug_css_alloc(struct cgroup_subsys_state *parent_css) +{ + struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL); + + if (!css) + return ERR_PTR(-ENOMEM); + + return css; +} + +static void debug_css_free(struct cgroup_subsys_state *css) +{ + kfree(css); +} + +static u64 debug_taskcount_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + return cgroup_task_count(css->cgroup); +} + +static u64 current_css_set_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + return (u64)(unsigned long)current->cgroups; +} + +static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + u64 count; + + rcu_read_lock(); + count = atomic_read(&task_css_set(current)->refcount); + rcu_read_unlock(); + return count; +} + +static int current_css_set_cg_links_read(struct seq_file *seq, void *v) +{ + struct cgrp_cset_link *link; + struct css_set *cset; + char *name_buf; + + name_buf = kmalloc(NAME_MAX + 1, GFP_KERNEL); + if (!name_buf) + return -ENOMEM; + + spin_lock_irq(&css_set_lock); + rcu_read_lock(); + cset = rcu_dereference(current->cgroups); + list_for_each_entry(link, &cset->cgrp_links, cgrp_link) { + struct cgroup *c = link->cgrp; + + cgroup_name(c, name_buf, NAME_MAX + 1); + seq_printf(seq, "Root %d group %s\n", + c->root->hierarchy_id, name_buf); + } + rcu_read_unlock(); + spin_unlock_irq(&css_set_lock); + kfree(name_buf); + return 0; +} + +#define MAX_TASKS_SHOWN_PER_CSS 25 +static int cgroup_css_links_read(struct seq_file *seq, void *v) +{ + struct cgroup_subsys_state *css = seq_css(seq); + struct cgrp_cset_link *link; + + spin_lock_irq(&css_set_lock); + list_for_each_entry(link, &css->cgroup->cset_links, cset_link) { + struct css_set *cset = link->cset; + struct task_struct *task; + int count = 0; + + seq_printf(seq, "css_set %p\n", cset); + + list_for_each_entry(task, &cset->tasks, cg_list) { + if (count++ > MAX_TASKS_SHOWN_PER_CSS) + goto overflow; + seq_printf(seq, " task %d\n", task_pid_vnr(task)); + } + + list_for_each_entry(task, &cset->mg_tasks, cg_list) { + if (count++ > MAX_TASKS_SHOWN_PER_CSS) + goto overflow; + seq_printf(seq, " task %d\n", task_pid_vnr(task)); + } + continue; + overflow: + seq_puts(seq, " ...\n"); + } + spin_unlock_irq(&css_set_lock); + return 0; +} + +static u64 releasable_read(struct cgroup_subsys_state *css, struct cftype *cft) +{ + return (!cgroup_is_populated(css->cgroup) && + !css_has_online_children(&css->cgroup->self)); +} + +static struct cftype debug_files[] = { + { + .name = "taskcount", + .read_u64 = debug_taskcount_read, + }, + + { + .name = "current_css_set", + .read_u64 = current_css_set_read, + }, + + { + .name = "current_css_set_refcount", + .read_u64 = current_css_set_refcount_read, + }, + + { + .name = "current_css_set_cg_links", + .seq_show = current_css_set_cg_links_read, + }, + + { + .name = "cgroup_css_links", + .seq_show = cgroup_css_links_read, + }, + + { + .name = "releasable", + .read_u64 = releasable_read, + }, + + { } /* terminate */ +}; + +struct cgroup_subsys debug_cgrp_subsys = { + .css_alloc = debug_css_alloc, + .css_free = debug_css_free, + .legacy_cftypes = debug_files, +}; +#endif /* CONFIG_CGROUP_DEBUG */ diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c new file mode 100644 index 000000000000..b3088886cd37 --- /dev/null +++ b/kernel/cgroup/cpuset.c @@ -0,0 +1,2752 @@ +/* + * kernel/cpuset.c + * + * Processor and Memory placement constraints for sets of tasks. + * + * Copyright (C) 2003 BULL SA. + * Copyright (C) 2004-2007 Silicon Graphics, Inc. + * Copyright (C) 2006 Google, Inc + * + * Portions derived from Patrick Mochel's sysfs code. + * sysfs is Copyright (c) 2001-3 Patrick Mochel + * + * 2003-10-10 Written by Simon Derr. + * 2003-10-22 Updates by Stephen Hemminger. + * 2004 May-July Rework by Paul Jackson. + * 2006 Rework by Paul Menage to use generic cgroups + * 2008 Rework of the scheduler domains and CPU hotplug handling + * by Max Krasnyansky + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file COPYING in the main directory of the Linux + * distribution for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key); + +/* See "Frequency meter" comments, below. */ + +struct fmeter { + int cnt; /* unprocessed events count */ + int val; /* most recent output value */ + time64_t time; /* clock (secs) when val computed */ + spinlock_t lock; /* guards read or write of above */ +}; + +struct cpuset { + struct cgroup_subsys_state css; + + unsigned long flags; /* "unsigned long" so bitops work */ + + /* + * On default hierarchy: + * + * The user-configured masks can only be changed by writing to + * cpuset.cpus and cpuset.mems, and won't be limited by the + * parent masks. + * + * The effective masks is the real masks that apply to the tasks + * in the cpuset. They may be changed if the configured masks are + * changed or hotplug happens. + * + * effective_mask == configured_mask & parent's effective_mask, + * and if it ends up empty, it will inherit the parent's mask. + * + * + * On legacy hierachy: + * + * The user-configured masks are always the same with effective masks. + */ + + /* user-configured CPUs and Memory Nodes allow to tasks */ + cpumask_var_t cpus_allowed; + nodemask_t mems_allowed; + + /* effective CPUs and Memory Nodes allow to tasks */ + cpumask_var_t effective_cpus; + nodemask_t effective_mems; + + /* + * This is old Memory Nodes tasks took on. + * + * - top_cpuset.old_mems_allowed is initialized to mems_allowed. + * - A new cpuset's old_mems_allowed is initialized when some + * task is moved into it. + * - old_mems_allowed is used in cpuset_migrate_mm() when we change + * cpuset.mems_allowed and have tasks' nodemask updated, and + * then old_mems_allowed is updated to mems_allowed. + */ + nodemask_t old_mems_allowed; + + struct fmeter fmeter; /* memory_pressure filter */ + + /* + * Tasks are being attached to this cpuset. Used to prevent + * zeroing cpus/mems_allowed between ->can_attach() and ->attach(). + */ + int attach_in_progress; + + /* partition number for rebuild_sched_domains() */ + int pn; + + /* for custom sched domain */ + int relax_domain_level; +}; + +static inline struct cpuset *css_cs(struct cgroup_subsys_state *css) +{ + return css ? container_of(css, struct cpuset, css) : NULL; +} + +/* Retrieve the cpuset for a task */ +static inline struct cpuset *task_cs(struct task_struct *task) +{ + return css_cs(task_css(task, cpuset_cgrp_id)); +} + +static inline struct cpuset *parent_cs(struct cpuset *cs) +{ + return css_cs(cs->css.parent); +} + +#ifdef CONFIG_NUMA +static inline bool task_has_mempolicy(struct task_struct *task) +{ + return task->mempolicy; +} +#else +static inline bool task_has_mempolicy(struct task_struct *task) +{ + return false; +} +#endif + + +/* bits in struct cpuset flags field */ +typedef enum { + CS_ONLINE, + CS_CPU_EXCLUSIVE, + CS_MEM_EXCLUSIVE, + CS_MEM_HARDWALL, + CS_MEMORY_MIGRATE, + CS_SCHED_LOAD_BALANCE, + CS_SPREAD_PAGE, + CS_SPREAD_SLAB, +} cpuset_flagbits_t; + +/* convenient tests for these bits */ +static inline bool is_cpuset_online(const struct cpuset *cs) +{ + return test_bit(CS_ONLINE, &cs->flags); +} + +static inline int is_cpu_exclusive(const struct cpuset *cs) +{ + return test_bit(CS_CPU_EXCLUSIVE, &cs->flags); +} + +static inline int is_mem_exclusive(const struct cpuset *cs) +{ + return test_bit(CS_MEM_EXCLUSIVE, &cs->flags); +} + +static inline int is_mem_hardwall(const struct cpuset *cs) +{ + return test_bit(CS_MEM_HARDWALL, &cs->flags); +} + +static inline int is_sched_load_balance(const struct cpuset *cs) +{ + return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); +} + +static inline int is_memory_migrate(const struct cpuset *cs) +{ + return test_bit(CS_MEMORY_MIGRATE, &cs->flags); +} + +static inline int is_spread_page(const struct cpuset *cs) +{ + return test_bit(CS_SPREAD_PAGE, &cs->flags); +} + +static inline int is_spread_slab(const struct cpuset *cs) +{ + return test_bit(CS_SPREAD_SLAB, &cs->flags); +} + +static struct cpuset top_cpuset = { + .flags = ((1 << CS_ONLINE) | (1 << CS_CPU_EXCLUSIVE) | + (1 << CS_MEM_EXCLUSIVE)), +}; + +/** + * cpuset_for_each_child - traverse online children of a cpuset + * @child_cs: loop cursor pointing to the current child + * @pos_css: used for iteration + * @parent_cs: target cpuset to walk children of + * + * Walk @child_cs through the online children of @parent_cs. Must be used + * with RCU read locked. + */ +#define cpuset_for_each_child(child_cs, pos_css, parent_cs) \ + css_for_each_child((pos_css), &(parent_cs)->css) \ + if (is_cpuset_online(((child_cs) = css_cs((pos_css))))) + +/** + * cpuset_for_each_descendant_pre - pre-order walk of a cpuset's descendants + * @des_cs: loop cursor pointing to the current descendant + * @pos_css: used for iteration + * @root_cs: target cpuset to walk ancestor of + * + * Walk @des_cs through the online descendants of @root_cs. Must be used + * with RCU read locked. The caller may modify @pos_css by calling + * css_rightmost_descendant() to skip subtree. @root_cs is included in the + * iteration and the first node to be visited. + */ +#define cpuset_for_each_descendant_pre(des_cs, pos_css, root_cs) \ + css_for_each_descendant_pre((pos_css), &(root_cs)->css) \ + if (is_cpuset_online(((des_cs) = css_cs((pos_css))))) + +/* + * There are two global locks guarding cpuset structures - cpuset_mutex and + * callback_lock. We also require taking task_lock() when dereferencing a + * task's cpuset pointer. See "The task_lock() exception", at the end of this + * comment. + * + * A task must hold both locks to modify cpusets. If a task holds + * cpuset_mutex, then it blocks others wanting that mutex, ensuring that it + * is the only task able to also acquire callback_lock and be able to + * modify cpusets. It can perform various checks on the cpuset structure + * first, knowing nothing will change. It can also allocate memory while + * just holding cpuset_mutex. While it is performing these checks, various + * callback routines can briefly acquire callback_lock to query cpusets. + * Once it is ready to make the changes, it takes callback_lock, blocking + * everyone else. + * + * Calls to the kernel memory allocator can not be made while holding + * callback_lock, as that would risk double tripping on callback_lock + * from one of the callbacks into the cpuset code from within + * __alloc_pages(). + * + * If a task is only holding callback_lock, then it has read-only + * access to cpusets. + * + * Now, the task_struct fields mems_allowed and mempolicy may be changed + * by other task, we use alloc_lock in the task_struct fields to protect + * them. + * + * The cpuset_common_file_read() handlers only hold callback_lock across + * small pieces of code, such as when reading out possibly multi-word + * cpumasks and nodemasks. + * + * Accessing a task's cpuset should be done in accordance with the + * guidelines for accessing subsystem state in kernel/cgroup.c + */ + +static DEFINE_MUTEX(cpuset_mutex); +static DEFINE_SPINLOCK(callback_lock); + +static struct workqueue_struct *cpuset_migrate_mm_wq; + +/* + * CPU / memory hotplug is handled asynchronously. + */ +static void cpuset_hotplug_workfn(struct work_struct *work); +static DECLARE_WORK(cpuset_hotplug_work, cpuset_hotplug_workfn); + +static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq); + +/* + * This is ugly, but preserves the userspace API for existing cpuset + * users. If someone tries to mount the "cpuset" filesystem, we + * silently switch it to mount "cgroup" instead + */ +static struct dentry *cpuset_mount(struct file_system_type *fs_type, + int flags, const char *unused_dev_name, void *data) +{ + struct file_system_type *cgroup_fs = get_fs_type("cgroup"); + struct dentry *ret = ERR_PTR(-ENODEV); + if (cgroup_fs) { + char mountopts[] = + "cpuset,noprefix," + "release_agent=/sbin/cpuset_release_agent"; + ret = cgroup_fs->mount(cgroup_fs, flags, + unused_dev_name, mountopts); + put_filesystem(cgroup_fs); + } + return ret; +} + +static struct file_system_type cpuset_fs_type = { + .name = "cpuset", + .mount = cpuset_mount, +}; + +/* + * Return in pmask the portion of a cpusets's cpus_allowed that + * are online. If none are online, walk up the cpuset hierarchy + * until we find one that does have some online cpus. + * + * One way or another, we guarantee to return some non-empty subset + * of cpu_online_mask. + * + * Call with callback_lock or cpuset_mutex held. + */ +static void guarantee_online_cpus(struct cpuset *cs, struct cpumask *pmask) +{ + while (!cpumask_intersects(cs->effective_cpus, cpu_online_mask)) { + cs = parent_cs(cs); + if (unlikely(!cs)) { + /* + * The top cpuset doesn't have any online cpu as a + * consequence of a race between cpuset_hotplug_work + * and cpu hotplug notifier. But we know the top + * cpuset's effective_cpus is on its way to to be + * identical to cpu_online_mask. + */ + cpumask_copy(pmask, cpu_online_mask); + return; + } + } + cpumask_and(pmask, cs->effective_cpus, cpu_online_mask); +} + +/* + * Return in *pmask the portion of a cpusets's mems_allowed that + * are online, with memory. If none are online with memory, walk + * up the cpuset hierarchy until we find one that does have some + * online mems. The top cpuset always has some mems online. + * + * One way or another, we guarantee to return some non-empty subset + * of node_states[N_MEMORY]. + * + * Call with callback_lock or cpuset_mutex held. + */ +static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask) +{ + while (!nodes_intersects(cs->effective_mems, node_states[N_MEMORY])) + cs = parent_cs(cs); + nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]); +} + +/* + * update task's spread flag if cpuset's page/slab spread flag is set + * + * Call with callback_lock or cpuset_mutex held. + */ +static void cpuset_update_task_spread_flag(struct cpuset *cs, + struct task_struct *tsk) +{ + if (is_spread_page(cs)) + task_set_spread_page(tsk); + else + task_clear_spread_page(tsk); + + if (is_spread_slab(cs)) + task_set_spread_slab(tsk); + else + task_clear_spread_slab(tsk); +} + +/* + * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q? + * + * One cpuset is a subset of another if all its allowed CPUs and + * Memory Nodes are a subset of the other, and its exclusive flags + * are only set if the other's are set. Call holding cpuset_mutex. + */ + +static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q) +{ + return cpumask_subset(p->cpus_allowed, q->cpus_allowed) && + nodes_subset(p->mems_allowed, q->mems_allowed) && + is_cpu_exclusive(p) <= is_cpu_exclusive(q) && + is_mem_exclusive(p) <= is_mem_exclusive(q); +} + +/** + * alloc_trial_cpuset - allocate a trial cpuset + * @cs: the cpuset that the trial cpuset duplicates + */ +static struct cpuset *alloc_trial_cpuset(struct cpuset *cs) +{ + struct cpuset *trial; + + trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL); + if (!trial) + return NULL; + + if (!alloc_cpumask_var(&trial->cpus_allowed, GFP_KERNEL)) + goto free_cs; + if (!alloc_cpumask_var(&trial->effective_cpus, GFP_KERNEL)) + goto free_cpus; + + cpumask_copy(trial->cpus_allowed, cs->cpus_allowed); + cpumask_copy(trial->effective_cpus, cs->effective_cpus); + return trial; + +free_cpus: + free_cpumask_var(trial->cpus_allowed); +free_cs: + kfree(trial); + return NULL; +} + +/** + * free_trial_cpuset - free the trial cpuset + * @trial: the trial cpuset to be freed + */ +static void free_trial_cpuset(struct cpuset *trial) +{ + free_cpumask_var(trial->effective_cpus); + free_cpumask_var(trial->cpus_allowed); + kfree(trial); +} + +/* + * validate_change() - Used to validate that any proposed cpuset change + * follows the structural rules for cpusets. + * + * If we replaced the flag and mask values of the current cpuset + * (cur) with those values in the trial cpuset (trial), would + * our various subset and exclusive rules still be valid? Presumes + * cpuset_mutex held. + * + * 'cur' is the address of an actual, in-use cpuset. Operations + * such as list traversal that depend on the actual address of the + * cpuset in the list must use cur below, not trial. + * + * 'trial' is the address of bulk structure copy of cur, with + * perhaps one or more of the fields cpus_allowed, mems_allowed, + * or flags changed to new, trial values. + * + * Return 0 if valid, -errno if not. + */ + +static int validate_change(struct cpuset *cur, struct cpuset *trial) +{ + struct cgroup_subsys_state *css; + struct cpuset *c, *par; + int ret; + + rcu_read_lock(); + + /* Each of our child cpusets must be a subset of us */ + ret = -EBUSY; + cpuset_for_each_child(c, css, cur) + if (!is_cpuset_subset(c, trial)) + goto out; + + /* Remaining checks don't apply to root cpuset */ + ret = 0; + if (cur == &top_cpuset) + goto out; + + par = parent_cs(cur); + + /* On legacy hiearchy, we must be a subset of our parent cpuset. */ + ret = -EACCES; + if (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && + !is_cpuset_subset(trial, par)) + goto out; + + /* + * If either I or some sibling (!= me) is exclusive, we can't + * overlap + */ + ret = -EINVAL; + cpuset_for_each_child(c, css, par) { + if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) && + c != cur && + cpumask_intersects(trial->cpus_allowed, c->cpus_allowed)) + goto out; + if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) && + c != cur && + nodes_intersects(trial->mems_allowed, c->mems_allowed)) + goto out; + } + + /* + * Cpusets with tasks - existing or newly being attached - can't + * be changed to have empty cpus_allowed or mems_allowed. + */ + ret = -ENOSPC; + if ((cgroup_is_populated(cur->css.cgroup) || cur->attach_in_progress)) { + if (!cpumask_empty(cur->cpus_allowed) && + cpumask_empty(trial->cpus_allowed)) + goto out; + if (!nodes_empty(cur->mems_allowed) && + nodes_empty(trial->mems_allowed)) + goto out; + } + + /* + * We can't shrink if we won't have enough room for SCHED_DEADLINE + * tasks. + */ + ret = -EBUSY; + if (is_cpu_exclusive(cur) && + !cpuset_cpumask_can_shrink(cur->cpus_allowed, + trial->cpus_allowed)) + goto out; + + ret = 0; +out: + rcu_read_unlock(); + return ret; +} + +#ifdef CONFIG_SMP +/* + * Helper routine for generate_sched_domains(). + * Do cpusets a, b have overlapping effective cpus_allowed masks? + */ +static int cpusets_overlap(struct cpuset *a, struct cpuset *b) +{ + return cpumask_intersects(a->effective_cpus, b->effective_cpus); +} + +static void +update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c) +{ + if (dattr->relax_domain_level < c->relax_domain_level) + dattr->relax_domain_level = c->relax_domain_level; + return; +} + +static void update_domain_attr_tree(struct sched_domain_attr *dattr, + struct cpuset *root_cs) +{ + struct cpuset *cp; + struct cgroup_subsys_state *pos_css; + + rcu_read_lock(); + cpuset_for_each_descendant_pre(cp, pos_css, root_cs) { + /* skip the whole subtree if @cp doesn't have any CPU */ + if (cpumask_empty(cp->cpus_allowed)) { + pos_css = css_rightmost_descendant(pos_css); + continue; + } + + if (is_sched_load_balance(cp)) + update_domain_attr(dattr, cp); + } + rcu_read_unlock(); +} + +/* + * generate_sched_domains() + * + * This function builds a partial partition of the systems CPUs + * A 'partial partition' is a set of non-overlapping subsets whose + * union is a subset of that set. + * The output of this function needs to be passed to kernel/sched/core.c + * partition_sched_domains() routine, which will rebuild the scheduler's + * load balancing domains (sched domains) as specified by that partial + * partition. + * + * See "What is sched_load_balance" in Documentation/cgroups/cpusets.txt + * for a background explanation of this. + * + * Does not return errors, on the theory that the callers of this + * routine would rather not worry about failures to rebuild sched + * domains when operating in the severe memory shortage situations + * that could cause allocation failures below. + * + * Must be called with cpuset_mutex held. + * + * The three key local variables below are: + * q - a linked-list queue of cpuset pointers, used to implement a + * top-down scan of all cpusets. This scan loads a pointer + * to each cpuset marked is_sched_load_balance into the + * array 'csa'. For our purposes, rebuilding the schedulers + * sched domains, we can ignore !is_sched_load_balance cpusets. + * csa - (for CpuSet Array) Array of pointers to all the cpusets + * that need to be load balanced, for convenient iterative + * access by the subsequent code that finds the best partition, + * i.e the set of domains (subsets) of CPUs such that the + * cpus_allowed of every cpuset marked is_sched_load_balance + * is a subset of one of these domains, while there are as + * many such domains as possible, each as small as possible. + * doms - Conversion of 'csa' to an array of cpumasks, for passing to + * the kernel/sched/core.c routine partition_sched_domains() in a + * convenient format, that can be easily compared to the prior + * value to determine what partition elements (sched domains) + * were changed (added or removed.) + * + * Finding the best partition (set of domains): + * The triple nested loops below over i, j, k scan over the + * load balanced cpusets (using the array of cpuset pointers in + * csa[]) looking for pairs of cpusets that have overlapping + * cpus_allowed, but which don't have the same 'pn' partition + * number and gives them in the same partition number. It keeps + * looping on the 'restart' label until it can no longer find + * any such pairs. + * + * The union of the cpus_allowed masks from the set of + * all cpusets having the same 'pn' value then form the one + * element of the partition (one sched domain) to be passed to + * partition_sched_domains(). + */ +static int generate_sched_domains(cpumask_var_t **domains, + struct sched_domain_attr **attributes) +{ + struct cpuset *cp; /* scans q */ + struct cpuset **csa; /* array of all cpuset ptrs */ + int csn; /* how many cpuset ptrs in csa so far */ + int i, j, k; /* indices for partition finding loops */ + cpumask_var_t *doms; /* resulting partition; i.e. sched domains */ + cpumask_var_t non_isolated_cpus; /* load balanced CPUs */ + struct sched_domain_attr *dattr; /* attributes for custom domains */ + int ndoms = 0; /* number of sched domains in result */ + int nslot; /* next empty doms[] struct cpumask slot */ + struct cgroup_subsys_state *pos_css; + + doms = NULL; + dattr = NULL; + csa = NULL; + + if (!alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL)) + goto done; + cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map); + + /* Special case for the 99% of systems with one, full, sched domain */ + if (is_sched_load_balance(&top_cpuset)) { + ndoms = 1; + doms = alloc_sched_domains(ndoms); + if (!doms) + goto done; + + dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL); + if (dattr) { + *dattr = SD_ATTR_INIT; + update_domain_attr_tree(dattr, &top_cpuset); + } + cpumask_and(doms[0], top_cpuset.effective_cpus, + non_isolated_cpus); + + goto done; + } + + csa = kmalloc(nr_cpusets() * sizeof(cp), GFP_KERNEL); + if (!csa) + goto done; + csn = 0; + + rcu_read_lock(); + cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) { + if (cp == &top_cpuset) + continue; + /* + * Continue traversing beyond @cp iff @cp has some CPUs and + * isn't load balancing. The former is obvious. The + * latter: All child cpusets contain a subset of the + * parent's cpus, so just skip them, and then we call + * update_domain_attr_tree() to calc relax_domain_level of + * the corresponding sched domain. + */ + if (!cpumask_empty(cp->cpus_allowed) && + !(is_sched_load_balance(cp) && + cpumask_intersects(cp->cpus_allowed, non_isolated_cpus))) + continue; + + if (is_sched_load_balance(cp)) + csa[csn++] = cp; + + /* skip @cp's subtree */ + pos_css = css_rightmost_descendant(pos_css); + } + rcu_read_unlock(); + + for (i = 0; i < csn; i++) + csa[i]->pn = i; + ndoms = csn; + +restart: + /* Find the best partition (set of sched domains) */ + for (i = 0; i < csn; i++) { + struct cpuset *a = csa[i]; + int apn = a->pn; + + for (j = 0; j < csn; j++) { + struct cpuset *b = csa[j]; + int bpn = b->pn; + + if (apn != bpn && cpusets_overlap(a, b)) { + for (k = 0; k < csn; k++) { + struct cpuset *c = csa[k]; + + if (c->pn == bpn) + c->pn = apn; + } + ndoms--; /* one less element */ + goto restart; + } + } + } + + /* + * Now we know how many domains to create. + * Convert to and populate cpu masks. + */ + doms = alloc_sched_domains(ndoms); + if (!doms) + goto done; + + /* + * The rest of the code, including the scheduler, can deal with + * dattr==NULL case. No need to abort if alloc fails. + */ + dattr = kmalloc(ndoms * sizeof(struct sched_domain_attr), GFP_KERNEL); + + for (nslot = 0, i = 0; i < csn; i++) { + struct cpuset *a = csa[i]; + struct cpumask *dp; + int apn = a->pn; + + if (apn < 0) { + /* Skip completed partitions */ + continue; + } + + dp = doms[nslot]; + + if (nslot == ndoms) { + static int warnings = 10; + if (warnings) { + pr_warn("rebuild_sched_domains confused: nslot %d, ndoms %d, csn %d, i %d, apn %d\n", + nslot, ndoms, csn, i, apn); + warnings--; + } + continue; + } + + cpumask_clear(dp); + if (dattr) + *(dattr + nslot) = SD_ATTR_INIT; + for (j = i; j < csn; j++) { + struct cpuset *b = csa[j]; + + if (apn == b->pn) { + cpumask_or(dp, dp, b->effective_cpus); + cpumask_and(dp, dp, non_isolated_cpus); + if (dattr) + update_domain_attr_tree(dattr + nslot, b); + + /* Done with this partition */ + b->pn = -1; + } + } + nslot++; + } + BUG_ON(nslot != ndoms); + +done: + free_cpumask_var(non_isolated_cpus); + kfree(csa); + + /* + * Fallback to the default domain if kmalloc() failed. + * See comments in partition_sched_domains(). + */ + if (doms == NULL) + ndoms = 1; + + *domains = doms; + *attributes = dattr; + return ndoms; +} + +/* + * Rebuild scheduler domains. + * + * If the flag 'sched_load_balance' of any cpuset with non-empty + * 'cpus' changes, or if the 'cpus' allowed changes in any cpuset + * which has that flag enabled, or if any cpuset with a non-empty + * 'cpus' is removed, then call this routine to rebuild the + * scheduler's dynamic sched domains. + * + * Call with cpuset_mutex held. Takes get_online_cpus(). + */ +static void rebuild_sched_domains_locked(void) +{ + struct sched_domain_attr *attr; + cpumask_var_t *doms; + int ndoms; + + lockdep_assert_held(&cpuset_mutex); + get_online_cpus(); + + /* + * We have raced with CPU hotplug. Don't do anything to avoid + * passing doms with offlined cpu to partition_sched_domains(). + * Anyways, hotplug work item will rebuild sched domains. + */ + if (!cpumask_equal(top_cpuset.effective_cpus, cpu_active_mask)) + goto out; + + /* Generate domain masks and attrs */ + ndoms = generate_sched_domains(&doms, &attr); + + /* Have scheduler rebuild the domains */ + partition_sched_domains(ndoms, doms, attr); +out: + put_online_cpus(); +} +#else /* !CONFIG_SMP */ +static void rebuild_sched_domains_locked(void) +{ +} +#endif /* CONFIG_SMP */ + +void rebuild_sched_domains(void) +{ + mutex_lock(&cpuset_mutex); + rebuild_sched_domains_locked(); + mutex_unlock(&cpuset_mutex); +} + +/** + * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset. + * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed + * + * Iterate through each task of @cs updating its cpus_allowed to the + * effective cpuset's. As this function is called with cpuset_mutex held, + * cpuset membership stays stable. + */ +static void update_tasks_cpumask(struct cpuset *cs) +{ + struct css_task_iter it; + struct task_struct *task; + + css_task_iter_start(&cs->css, &it); + while ((task = css_task_iter_next(&it))) + set_cpus_allowed_ptr(task, cs->effective_cpus); + css_task_iter_end(&it); +} + +/* + * update_cpumasks_hier - Update effective cpumasks and tasks in the subtree + * @cs: the cpuset to consider + * @new_cpus: temp variable for calculating new effective_cpus + * + * When congifured cpumask is changed, the effective cpumasks of this cpuset + * and all its descendants need to be updated. + * + * On legacy hierachy, effective_cpus will be the same with cpu_allowed. + * + * Called with cpuset_mutex held + */ +static void update_cpumasks_hier(struct cpuset *cs, struct cpumask *new_cpus) +{ + struct cpuset *cp; + struct cgroup_subsys_state *pos_css; + bool need_rebuild_sched_domains = false; + + rcu_read_lock(); + cpuset_for_each_descendant_pre(cp, pos_css, cs) { + struct cpuset *parent = parent_cs(cp); + + cpumask_and(new_cpus, cp->cpus_allowed, parent->effective_cpus); + + /* + * If it becomes empty, inherit the effective mask of the + * parent, which is guaranteed to have some CPUs. + */ + if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && + cpumask_empty(new_cpus)) + cpumask_copy(new_cpus, parent->effective_cpus); + + /* Skip the whole subtree if the cpumask remains the same. */ + if (cpumask_equal(new_cpus, cp->effective_cpus)) { + pos_css = css_rightmost_descendant(pos_css); + continue; + } + + if (!css_tryget_online(&cp->css)) + continue; + rcu_read_unlock(); + + spin_lock_irq(&callback_lock); + cpumask_copy(cp->effective_cpus, new_cpus); + spin_unlock_irq(&callback_lock); + + WARN_ON(!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && + !cpumask_equal(cp->cpus_allowed, cp->effective_cpus)); + + update_tasks_cpumask(cp); + + /* + * If the effective cpumask of any non-empty cpuset is changed, + * we need to rebuild sched domains. + */ + if (!cpumask_empty(cp->cpus_allowed) && + is_sched_load_balance(cp)) + need_rebuild_sched_domains = true; + + rcu_read_lock(); + css_put(&cp->css); + } + rcu_read_unlock(); + + if (need_rebuild_sched_domains) + rebuild_sched_domains_locked(); +} + +/** + * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it + * @cs: the cpuset to consider + * @trialcs: trial cpuset + * @buf: buffer of cpu numbers written to this cpuset + */ +static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs, + const char *buf) +{ + int retval; + + /* top_cpuset.cpus_allowed tracks cpu_online_mask; it's read-only */ + if (cs == &top_cpuset) + return -EACCES; + + /* + * An empty cpus_allowed is ok only if the cpuset has no tasks. + * Since cpulist_parse() fails on an empty mask, we special case + * that parsing. The validate_change() call ensures that cpusets + * with tasks have cpus. + */ + if (!*buf) { + cpumask_clear(trialcs->cpus_allowed); + } else { + retval = cpulist_parse(buf, trialcs->cpus_allowed); + if (retval < 0) + return retval; + + if (!cpumask_subset(trialcs->cpus_allowed, + top_cpuset.cpus_allowed)) + return -EINVAL; + } + + /* Nothing to do if the cpus didn't change */ + if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed)) + return 0; + + retval = validate_change(cs, trialcs); + if (retval < 0) + return retval; + + spin_lock_irq(&callback_lock); + cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed); + spin_unlock_irq(&callback_lock); + + /* use trialcs->cpus_allowed as a temp variable */ + update_cpumasks_hier(cs, trialcs->cpus_allowed); + return 0; +} + +/* + * Migrate memory region from one set of nodes to another. This is + * performed asynchronously as it can be called from process migration path + * holding locks involved in process management. All mm migrations are + * performed in the queued order and can be waited for by flushing + * cpuset_migrate_mm_wq. + */ + +struct cpuset_migrate_mm_work { + struct work_struct work; + struct mm_struct *mm; + nodemask_t from; + nodemask_t to; +}; + +static void cpuset_migrate_mm_workfn(struct work_struct *work) +{ + struct cpuset_migrate_mm_work *mwork = + container_of(work, struct cpuset_migrate_mm_work, work); + + /* on a wq worker, no need to worry about %current's mems_allowed */ + do_migrate_pages(mwork->mm, &mwork->from, &mwork->to, MPOL_MF_MOVE_ALL); + mmput(mwork->mm); + kfree(mwork); +} + +static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from, + const nodemask_t *to) +{ + struct cpuset_migrate_mm_work *mwork; + + mwork = kzalloc(sizeof(*mwork), GFP_KERNEL); + if (mwork) { + mwork->mm = mm; + mwork->from = *from; + mwork->to = *to; + INIT_WORK(&mwork->work, cpuset_migrate_mm_workfn); + queue_work(cpuset_migrate_mm_wq, &mwork->work); + } else { + mmput(mm); + } +} + +static void cpuset_post_attach(void) +{ + flush_workqueue(cpuset_migrate_mm_wq); +} + +/* + * cpuset_change_task_nodemask - change task's mems_allowed and mempolicy + * @tsk: the task to change + * @newmems: new nodes that the task will be set + * + * In order to avoid seeing no nodes if the old and new nodes are disjoint, + * we structure updates as setting all new allowed nodes, then clearing newly + * disallowed ones. + */ +static void cpuset_change_task_nodemask(struct task_struct *tsk, + nodemask_t *newmems) +{ + bool need_loop; + + task_lock(tsk); + /* + * Determine if a loop is necessary if another thread is doing + * read_mems_allowed_begin(). If at least one node remains unchanged and + * tsk does not have a mempolicy, then an empty nodemask will not be + * possible when mems_allowed is larger than a word. + */ + need_loop = task_has_mempolicy(tsk) || + !nodes_intersects(*newmems, tsk->mems_allowed); + + if (need_loop) { + local_irq_disable(); + write_seqcount_begin(&tsk->mems_allowed_seq); + } + + nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems); + mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP1); + + mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP2); + tsk->mems_allowed = *newmems; + + if (need_loop) { + write_seqcount_end(&tsk->mems_allowed_seq); + local_irq_enable(); + } + + task_unlock(tsk); +} + +static void *cpuset_being_rebound; + +/** + * update_tasks_nodemask - Update the nodemasks of tasks in the cpuset. + * @cs: the cpuset in which each task's mems_allowed mask needs to be changed + * + * Iterate through each task of @cs updating its mems_allowed to the + * effective cpuset's. As this function is called with cpuset_mutex held, + * cpuset membership stays stable. + */ +static void update_tasks_nodemask(struct cpuset *cs) +{ + static nodemask_t newmems; /* protected by cpuset_mutex */ + struct css_task_iter it; + struct task_struct *task; + + cpuset_being_rebound = cs; /* causes mpol_dup() rebind */ + + guarantee_online_mems(cs, &newmems); + + /* + * The mpol_rebind_mm() call takes mmap_sem, which we couldn't + * take while holding tasklist_lock. Forks can happen - the + * mpol_dup() cpuset_being_rebound check will catch such forks, + * and rebind their vma mempolicies too. Because we still hold + * the global cpuset_mutex, we know that no other rebind effort + * will be contending for the global variable cpuset_being_rebound. + * It's ok if we rebind the same mm twice; mpol_rebind_mm() + * is idempotent. Also migrate pages in each mm to new nodes. + */ + css_task_iter_start(&cs->css, &it); + while ((task = css_task_iter_next(&it))) { + struct mm_struct *mm; + bool migrate; + + cpuset_change_task_nodemask(task, &newmems); + + mm = get_task_mm(task); + if (!mm) + continue; + + migrate = is_memory_migrate(cs); + + mpol_rebind_mm(mm, &cs->mems_allowed); + if (migrate) + cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems); + else + mmput(mm); + } + css_task_iter_end(&it); + + /* + * All the tasks' nodemasks have been updated, update + * cs->old_mems_allowed. + */ + cs->old_mems_allowed = newmems; + + /* We're done rebinding vmas to this cpuset's new mems_allowed. */ + cpuset_being_rebound = NULL; +} + +/* + * update_nodemasks_hier - Update effective nodemasks and tasks in the subtree + * @cs: the cpuset to consider + * @new_mems: a temp variable for calculating new effective_mems + * + * When configured nodemask is changed, the effective nodemasks of this cpuset + * and all its descendants need to be updated. + * + * On legacy hiearchy, effective_mems will be the same with mems_allowed. + * + * Called with cpuset_mutex held + */ +static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems) +{ + struct cpuset *cp; + struct cgroup_subsys_state *pos_css; + + rcu_read_lock(); + cpuset_for_each_descendant_pre(cp, pos_css, cs) { + struct cpuset *parent = parent_cs(cp); + + nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems); + + /* + * If it becomes empty, inherit the effective mask of the + * parent, which is guaranteed to have some MEMs. + */ + if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && + nodes_empty(*new_mems)) + *new_mems = parent->effective_mems; + + /* Skip the whole subtree if the nodemask remains the same. */ + if (nodes_equal(*new_mems, cp->effective_mems)) { + pos_css = css_rightmost_descendant(pos_css); + continue; + } + + if (!css_tryget_online(&cp->css)) + continue; + rcu_read_unlock(); + + spin_lock_irq(&callback_lock); + cp->effective_mems = *new_mems; + spin_unlock_irq(&callback_lock); + + WARN_ON(!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && + !nodes_equal(cp->mems_allowed, cp->effective_mems)); + + update_tasks_nodemask(cp); + + rcu_read_lock(); + css_put(&cp->css); + } + rcu_read_unlock(); +} + +/* + * Handle user request to change the 'mems' memory placement + * of a cpuset. Needs to validate the request, update the + * cpusets mems_allowed, and for each task in the cpuset, + * update mems_allowed and rebind task's mempolicy and any vma + * mempolicies and if the cpuset is marked 'memory_migrate', + * migrate the tasks pages to the new memory. + * + * Call with cpuset_mutex held. May take callback_lock during call. + * Will take tasklist_lock, scan tasklist for tasks in cpuset cs, + * lock each such tasks mm->mmap_sem, scan its vma's and rebind + * their mempolicies to the cpusets new mems_allowed. + */ +static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs, + const char *buf) +{ + int retval; + + /* + * top_cpuset.mems_allowed tracks node_stats[N_MEMORY]; + * it's read-only + */ + if (cs == &top_cpuset) { + retval = -EACCES; + goto done; + } + + /* + * An empty mems_allowed is ok iff there are no tasks in the cpuset. + * Since nodelist_parse() fails on an empty mask, we special case + * that parsing. The validate_change() call ensures that cpusets + * with tasks have memory. + */ + if (!*buf) { + nodes_clear(trialcs->mems_allowed); + } else { + retval = nodelist_parse(buf, trialcs->mems_allowed); + if (retval < 0) + goto done; + + if (!nodes_subset(trialcs->mems_allowed, + top_cpuset.mems_allowed)) { + retval = -EINVAL; + goto done; + } + } + + if (nodes_equal(cs->mems_allowed, trialcs->mems_allowed)) { + retval = 0; /* Too easy - nothing to do */ + goto done; + } + retval = validate_change(cs, trialcs); + if (retval < 0) + goto done; + + spin_lock_irq(&callback_lock); + cs->mems_allowed = trialcs->mems_allowed; + spin_unlock_irq(&callback_lock); + + /* use trialcs->mems_allowed as a temp variable */ + update_nodemasks_hier(cs, &trialcs->mems_allowed); +done: + return retval; +} + +int current_cpuset_is_being_rebound(void) +{ + int ret; + + rcu_read_lock(); + ret = task_cs(current) == cpuset_being_rebound; + rcu_read_unlock(); + + return ret; +} + +static int update_relax_domain_level(struct cpuset *cs, s64 val) +{ +#ifdef CONFIG_SMP + if (val < -1 || val >= sched_domain_level_max) + return -EINVAL; +#endif + + if (val != cs->relax_domain_level) { + cs->relax_domain_level = val; + if (!cpumask_empty(cs->cpus_allowed) && + is_sched_load_balance(cs)) + rebuild_sched_domains_locked(); + } + + return 0; +} + +/** + * update_tasks_flags - update the spread flags of tasks in the cpuset. + * @cs: the cpuset in which each task's spread flags needs to be changed + * + * Iterate through each task of @cs updating its spread flags. As this + * function is called with cpuset_mutex held, cpuset membership stays + * stable. + */ +static void update_tasks_flags(struct cpuset *cs) +{ + struct css_task_iter it; + struct task_struct *task; + + css_task_iter_start(&cs->css, &it); + while ((task = css_task_iter_next(&it))) + cpuset_update_task_spread_flag(cs, task); + css_task_iter_end(&it); +} + +/* + * update_flag - read a 0 or a 1 in a file and update associated flag + * bit: the bit to update (see cpuset_flagbits_t) + * cs: the cpuset to update + * turning_on: whether the flag is being set or cleared + * + * Call with cpuset_mutex held. + */ + +static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs, + int turning_on) +{ + struct cpuset *trialcs; + int balance_flag_changed; + int spread_flag_changed; + int err; + + trialcs = alloc_trial_cpuset(cs); + if (!trialcs) + return -ENOMEM; + + if (turning_on) + set_bit(bit, &trialcs->flags); + else + clear_bit(bit, &trialcs->flags); + + err = validate_change(cs, trialcs); + if (err < 0) + goto out; + + balance_flag_changed = (is_sched_load_balance(cs) != + is_sched_load_balance(trialcs)); + + spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs)) + || (is_spread_page(cs) != is_spread_page(trialcs))); + + spin_lock_irq(&callback_lock); + cs->flags = trialcs->flags; + spin_unlock_irq(&callback_lock); + + if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed) + rebuild_sched_domains_locked(); + + if (spread_flag_changed) + update_tasks_flags(cs); +out: + free_trial_cpuset(trialcs); + return err; +} + +/* + * Frequency meter - How fast is some event occurring? + * + * These routines manage a digitally filtered, constant time based, + * event frequency meter. There are four routines: + * fmeter_init() - initialize a frequency meter. + * fmeter_markevent() - called each time the event happens. + * fmeter_getrate() - returns the recent rate of such events. + * fmeter_update() - internal routine used to update fmeter. + * + * A common data structure is passed to each of these routines, + * which is used to keep track of the state required to manage the + * frequency meter and its digital filter. + * + * The filter works on the number of events marked per unit time. + * The filter is single-pole low-pass recursive (IIR). The time unit + * is 1 second. Arithmetic is done using 32-bit integers scaled to + * simulate 3 decimal digits of precision (multiplied by 1000). + * + * With an FM_COEF of 933, and a time base of 1 second, the filter + * has a half-life of 10 seconds, meaning that if the events quit + * happening, then the rate returned from the fmeter_getrate() + * will be cut in half each 10 seconds, until it converges to zero. + * + * It is not worth doing a real infinitely recursive filter. If more + * than FM_MAXTICKS ticks have elapsed since the last filter event, + * just compute FM_MAXTICKS ticks worth, by which point the level + * will be stable. + * + * Limit the count of unprocessed events to FM_MAXCNT, so as to avoid + * arithmetic overflow in the fmeter_update() routine. + * + * Given the simple 32 bit integer arithmetic used, this meter works + * best for reporting rates between one per millisecond (msec) and + * one per 32 (approx) seconds. At constant rates faster than one + * per msec it maxes out at values just under 1,000,000. At constant + * rates between one per msec, and one per second it will stabilize + * to a value N*1000, where N is the rate of events per second. + * At constant rates between one per second and one per 32 seconds, + * it will be choppy, moving up on the seconds that have an event, + * and then decaying until the next event. At rates slower than + * about one in 32 seconds, it decays all the way back to zero between + * each event. + */ + +#define FM_COEF 933 /* coefficient for half-life of 10 secs */ +#define FM_MAXTICKS ((u32)99) /* useless computing more ticks than this */ +#define FM_MAXCNT 1000000 /* limit cnt to avoid overflow */ +#define FM_SCALE 1000 /* faux fixed point scale */ + +/* Initialize a frequency meter */ +static void fmeter_init(struct fmeter *fmp) +{ + fmp->cnt = 0; + fmp->val = 0; + fmp->time = 0; + spin_lock_init(&fmp->lock); +} + +/* Internal meter update - process cnt events and update value */ +static void fmeter_update(struct fmeter *fmp) +{ + time64_t now; + u32 ticks; + + now = ktime_get_seconds(); + ticks = now - fmp->time; + + if (ticks == 0) + return; + + ticks = min(FM_MAXTICKS, ticks); + while (ticks-- > 0) + fmp->val = (FM_COEF * fmp->val) / FM_SCALE; + fmp->time = now; + + fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE; + fmp->cnt = 0; +} + +/* Process any previous ticks, then bump cnt by one (times scale). */ +static void fmeter_markevent(struct fmeter *fmp) +{ + spin_lock(&fmp->lock); + fmeter_update(fmp); + fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE); + spin_unlock(&fmp->lock); +} + +/* Process any previous ticks, then return current value. */ +static int fmeter_getrate(struct fmeter *fmp) +{ + int val; + + spin_lock(&fmp->lock); + fmeter_update(fmp); + val = fmp->val; + spin_unlock(&fmp->lock); + return val; +} + +static struct cpuset *cpuset_attach_old_cs; + +/* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */ +static int cpuset_can_attach(struct cgroup_taskset *tset) +{ + struct cgroup_subsys_state *css; + struct cpuset *cs; + struct task_struct *task; + int ret; + + /* used later by cpuset_attach() */ + cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset, &css)); + cs = css_cs(css); + + mutex_lock(&cpuset_mutex); + + /* allow moving tasks into an empty cpuset if on default hierarchy */ + ret = -ENOSPC; + if (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && + (cpumask_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed))) + goto out_unlock; + + cgroup_taskset_for_each(task, css, tset) { + ret = task_can_attach(task, cs->cpus_allowed); + if (ret) + goto out_unlock; + ret = security_task_setscheduler(task); + if (ret) + goto out_unlock; + } + + /* + * Mark attach is in progress. This makes validate_change() fail + * changes which zero cpus/mems_allowed. + */ + cs->attach_in_progress++; + ret = 0; +out_unlock: + mutex_unlock(&cpuset_mutex); + return ret; +} + +static void cpuset_cancel_attach(struct cgroup_taskset *tset) +{ + struct cgroup_subsys_state *css; + struct cpuset *cs; + + cgroup_taskset_first(tset, &css); + cs = css_cs(css); + + mutex_lock(&cpuset_mutex); + css_cs(css)->attach_in_progress--; + mutex_unlock(&cpuset_mutex); +} + +/* + * Protected by cpuset_mutex. cpus_attach is used only by cpuset_attach() + * but we can't allocate it dynamically there. Define it global and + * allocate from cpuset_init(). + */ +static cpumask_var_t cpus_attach; + +static void cpuset_attach(struct cgroup_taskset *tset) +{ + /* static buf protected by cpuset_mutex */ + static nodemask_t cpuset_attach_nodemask_to; + struct task_struct *task; + struct task_struct *leader; + struct cgroup_subsys_state *css; + struct cpuset *cs; + struct cpuset *oldcs = cpuset_attach_old_cs; + + cgroup_taskset_first(tset, &css); + cs = css_cs(css); + + mutex_lock(&cpuset_mutex); + + /* prepare for attach */ + if (cs == &top_cpuset) + cpumask_copy(cpus_attach, cpu_possible_mask); + else + guarantee_online_cpus(cs, cpus_attach); + + guarantee_online_mems(cs, &cpuset_attach_nodemask_to); + + cgroup_taskset_for_each(task, css, tset) { + /* + * can_attach beforehand should guarantee that this doesn't + * fail. TODO: have a better way to handle failure here + */ + WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach)); + + cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to); + cpuset_update_task_spread_flag(cs, task); + } + + /* + * Change mm for all threadgroup leaders. This is expensive and may + * sleep and should be moved outside migration path proper. + */ + cpuset_attach_nodemask_to = cs->effective_mems; + cgroup_taskset_for_each_leader(leader, css, tset) { + struct mm_struct *mm = get_task_mm(leader); + + if (mm) { + mpol_rebind_mm(mm, &cpuset_attach_nodemask_to); + + /* + * old_mems_allowed is the same with mems_allowed + * here, except if this task is being moved + * automatically due to hotplug. In that case + * @mems_allowed has been updated and is empty, so + * @old_mems_allowed is the right nodesets that we + * migrate mm from. + */ + if (is_memory_migrate(cs)) + cpuset_migrate_mm(mm, &oldcs->old_mems_allowed, + &cpuset_attach_nodemask_to); + else + mmput(mm); + } + } + + cs->old_mems_allowed = cpuset_attach_nodemask_to; + + cs->attach_in_progress--; + if (!cs->attach_in_progress) + wake_up(&cpuset_attach_wq); + + mutex_unlock(&cpuset_mutex); +} + +/* The various types of files and directories in a cpuset file system */ + +typedef enum { + FILE_MEMORY_MIGRATE, + FILE_CPULIST, + FILE_MEMLIST, + FILE_EFFECTIVE_CPULIST, + FILE_EFFECTIVE_MEMLIST, + FILE_CPU_EXCLUSIVE, + FILE_MEM_EXCLUSIVE, + FILE_MEM_HARDWALL, + FILE_SCHED_LOAD_BALANCE, + FILE_SCHED_RELAX_DOMAIN_LEVEL, + FILE_MEMORY_PRESSURE_ENABLED, + FILE_MEMORY_PRESSURE, + FILE_SPREAD_PAGE, + FILE_SPREAD_SLAB, +} cpuset_filetype_t; + +static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft, + u64 val) +{ + struct cpuset *cs = css_cs(css); + cpuset_filetype_t type = cft->private; + int retval = 0; + + mutex_lock(&cpuset_mutex); + if (!is_cpuset_online(cs)) { + retval = -ENODEV; + goto out_unlock; + } + + switch (type) { + case FILE_CPU_EXCLUSIVE: + retval = update_flag(CS_CPU_EXCLUSIVE, cs, val); + break; + case FILE_MEM_EXCLUSIVE: + retval = update_flag(CS_MEM_EXCLUSIVE, cs, val); + break; + case FILE_MEM_HARDWALL: + retval = update_flag(CS_MEM_HARDWALL, cs, val); + break; + case FILE_SCHED_LOAD_BALANCE: + retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, val); + break; + case FILE_MEMORY_MIGRATE: + retval = update_flag(CS_MEMORY_MIGRATE, cs, val); + break; + case FILE_MEMORY_PRESSURE_ENABLED: + cpuset_memory_pressure_enabled = !!val; + break; + case FILE_SPREAD_PAGE: + retval = update_flag(CS_SPREAD_PAGE, cs, val); + break; + case FILE_SPREAD_SLAB: + retval = update_flag(CS_SPREAD_SLAB, cs, val); + break; + default: + retval = -EINVAL; + break; + } +out_unlock: + mutex_unlock(&cpuset_mutex); + return retval; +} + +static int cpuset_write_s64(struct cgroup_subsys_state *css, struct cftype *cft, + s64 val) +{ + struct cpuset *cs = css_cs(css); + cpuset_filetype_t type = cft->private; + int retval = -ENODEV; + + mutex_lock(&cpuset_mutex); + if (!is_cpuset_online(cs)) + goto out_unlock; + + switch (type) { + case FILE_SCHED_RELAX_DOMAIN_LEVEL: + retval = update_relax_domain_level(cs, val); + break; + default: + retval = -EINVAL; + break; + } +out_unlock: + mutex_unlock(&cpuset_mutex); + return retval; +} + +/* + * Common handling for a write to a "cpus" or "mems" file. + */ +static ssize_t cpuset_write_resmask(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct cpuset *cs = css_cs(of_css(of)); + struct cpuset *trialcs; + int retval = -ENODEV; + + buf = strstrip(buf); + + /* + * CPU or memory hotunplug may leave @cs w/o any execution + * resources, in which case the hotplug code asynchronously updates + * configuration and transfers all tasks to the nearest ancestor + * which can execute. + * + * As writes to "cpus" or "mems" may restore @cs's execution + * resources, wait for the previously scheduled operations before + * proceeding, so that we don't end up keep removing tasks added + * after execution capability is restored. + * + * cpuset_hotplug_work calls back into cgroup core via + * cgroup_transfer_tasks() and waiting for it from a cgroupfs + * operation like this one can lead to a deadlock through kernfs + * active_ref protection. Let's break the protection. Losing the + * protection is okay as we check whether @cs is online after + * grabbing cpuset_mutex anyway. This only happens on the legacy + * hierarchies. + */ + css_get(&cs->css); + kernfs_break_active_protection(of->kn); + flush_work(&cpuset_hotplug_work); + + mutex_lock(&cpuset_mutex); + if (!is_cpuset_online(cs)) + goto out_unlock; + + trialcs = alloc_trial_cpuset(cs); + if (!trialcs) { + retval = -ENOMEM; + goto out_unlock; + } + + switch (of_cft(of)->private) { + case FILE_CPULIST: + retval = update_cpumask(cs, trialcs, buf); + break; + case FILE_MEMLIST: + retval = update_nodemask(cs, trialcs, buf); + break; + default: + retval = -EINVAL; + break; + } + + free_trial_cpuset(trialcs); +out_unlock: + mutex_unlock(&cpuset_mutex); + kernfs_unbreak_active_protection(of->kn); + css_put(&cs->css); + flush_workqueue(cpuset_migrate_mm_wq); + return retval ?: nbytes; +} + +/* + * These ascii lists should be read in a single call, by using a user + * buffer large enough to hold the entire map. If read in smaller + * chunks, there is no guarantee of atomicity. Since the display format + * used, list of ranges of sequential numbers, is variable length, + * and since these maps can change value dynamically, one could read + * gibberish by doing partial reads while a list was changing. + */ +static int cpuset_common_seq_show(struct seq_file *sf, void *v) +{ + struct cpuset *cs = css_cs(seq_css(sf)); + cpuset_filetype_t type = seq_cft(sf)->private; + int ret = 0; + + spin_lock_irq(&callback_lock); + + switch (type) { + case FILE_CPULIST: + seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->cpus_allowed)); + break; + case FILE_MEMLIST: + seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->mems_allowed)); + break; + case FILE_EFFECTIVE_CPULIST: + seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_cpus)); + break; + case FILE_EFFECTIVE_MEMLIST: + seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->effective_mems)); + break; + default: + ret = -EINVAL; + } + + spin_unlock_irq(&callback_lock); + return ret; +} + +static u64 cpuset_read_u64(struct cgroup_subsys_state *css, struct cftype *cft) +{ + struct cpuset *cs = css_cs(css); + cpuset_filetype_t type = cft->private; + switch (type) { + case FILE_CPU_EXCLUSIVE: + return is_cpu_exclusive(cs); + case FILE_MEM_EXCLUSIVE: + return is_mem_exclusive(cs); + case FILE_MEM_HARDWALL: + return is_mem_hardwall(cs); + case FILE_SCHED_LOAD_BALANCE: + return is_sched_load_balance(cs); + case FILE_MEMORY_MIGRATE: + return is_memory_migrate(cs); + case FILE_MEMORY_PRESSURE_ENABLED: + return cpuset_memory_pressure_enabled; + case FILE_MEMORY_PRESSURE: + return fmeter_getrate(&cs->fmeter); + case FILE_SPREAD_PAGE: + return is_spread_page(cs); + case FILE_SPREAD_SLAB: + return is_spread_slab(cs); + default: + BUG(); + } + + /* Unreachable but makes gcc happy */ + return 0; +} + +static s64 cpuset_read_s64(struct cgroup_subsys_state *css, struct cftype *cft) +{ + struct cpuset *cs = css_cs(css); + cpuset_filetype_t type = cft->private; + switch (type) { + case FILE_SCHED_RELAX_DOMAIN_LEVEL: + return cs->relax_domain_level; + default: + BUG(); + } + + /* Unrechable but makes gcc happy */ + return 0; +} + + +/* + * for the common functions, 'private' gives the type of file + */ + +static struct cftype files[] = { + { + .name = "cpus", + .seq_show = cpuset_common_seq_show, + .write = cpuset_write_resmask, + .max_write_len = (100U + 6 * NR_CPUS), + .private = FILE_CPULIST, + }, + + { + .name = "mems", + .seq_show = cpuset_common_seq_show, + .write = cpuset_write_resmask, + .max_write_len = (100U + 6 * MAX_NUMNODES), + .private = FILE_MEMLIST, + }, + + { + .name = "effective_cpus", + .seq_show = cpuset_common_seq_show, + .private = FILE_EFFECTIVE_CPULIST, + }, + + { + .name = "effective_mems", + .seq_show = cpuset_common_seq_show, + .private = FILE_EFFECTIVE_MEMLIST, + }, + + { + .name = "cpu_exclusive", + .read_u64 = cpuset_read_u64, + .write_u64 = cpuset_write_u64, + .private = FILE_CPU_EXCLUSIVE, + }, + + { + .name = "mem_exclusive", + .read_u64 = cpuset_read_u64, + .write_u64 = cpuset_write_u64, + .private = FILE_MEM_EXCLUSIVE, + }, + + { + .name = "mem_hardwall", + .read_u64 = cpuset_read_u64, + .write_u64 = cpuset_write_u64, + .private = FILE_MEM_HARDWALL, + }, + + { + .name = "sched_load_balance", + .read_u64 = cpuset_read_u64, + .write_u64 = cpuset_write_u64, + .private = FILE_SCHED_LOAD_BALANCE, + }, + + { + .name = "sched_relax_domain_level", + .read_s64 = cpuset_read_s64, + .write_s64 = cpuset_write_s64, + .private = FILE_SCHED_RELAX_DOMAIN_LEVEL, + }, + + { + .name = "memory_migrate", + .read_u64 = cpuset_read_u64, + .write_u64 = cpuset_write_u64, + .private = FILE_MEMORY_MIGRATE, + }, + + { + .name = "memory_pressure", + .read_u64 = cpuset_read_u64, + }, + + { + .name = "memory_spread_page", + .read_u64 = cpuset_read_u64, + .write_u64 = cpuset_write_u64, + .private = FILE_SPREAD_PAGE, + }, + + { + .name = "memory_spread_slab", + .read_u64 = cpuset_read_u64, + .write_u64 = cpuset_write_u64, + .private = FILE_SPREAD_SLAB, + }, + + { + .name = "memory_pressure_enabled", + .flags = CFTYPE_ONLY_ON_ROOT, + .read_u64 = cpuset_read_u64, + .write_u64 = cpuset_write_u64, + .private = FILE_MEMORY_PRESSURE_ENABLED, + }, + + { } /* terminate */ +}; + +/* + * cpuset_css_alloc - allocate a cpuset css + * cgrp: control group that the new cpuset will be part of + */ + +static struct cgroup_subsys_state * +cpuset_css_alloc(struct cgroup_subsys_state *parent_css) +{ + struct cpuset *cs; + + if (!parent_css) + return &top_cpuset.css; + + cs = kzalloc(sizeof(*cs), GFP_KERNEL); + if (!cs) + return ERR_PTR(-ENOMEM); + if (!alloc_cpumask_var(&cs->cpus_allowed, GFP_KERNEL)) + goto free_cs; + if (!alloc_cpumask_var(&cs->effective_cpus, GFP_KERNEL)) + goto free_cpus; + + set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); + cpumask_clear(cs->cpus_allowed); + nodes_clear(cs->mems_allowed); + cpumask_clear(cs->effective_cpus); + nodes_clear(cs->effective_mems); + fmeter_init(&cs->fmeter); + cs->relax_domain_level = -1; + + return &cs->css; + +free_cpus: + free_cpumask_var(cs->cpus_allowed); +free_cs: + kfree(cs); + return ERR_PTR(-ENOMEM); +} + +static int cpuset_css_online(struct cgroup_subsys_state *css) +{ + struct cpuset *cs = css_cs(css); + struct cpuset *parent = parent_cs(cs); + struct cpuset *tmp_cs; + struct cgroup_subsys_state *pos_css; + + if (!parent) + return 0; + + mutex_lock(&cpuset_mutex); + + set_bit(CS_ONLINE, &cs->flags); + if (is_spread_page(parent)) + set_bit(CS_SPREAD_PAGE, &cs->flags); + if (is_spread_slab(parent)) + set_bit(CS_SPREAD_SLAB, &cs->flags); + + cpuset_inc(); + + spin_lock_irq(&callback_lock); + if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys)) { + cpumask_copy(cs->effective_cpus, parent->effective_cpus); + cs->effective_mems = parent->effective_mems; + } + spin_unlock_irq(&callback_lock); + + if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags)) + goto out_unlock; + + /* + * Clone @parent's configuration if CGRP_CPUSET_CLONE_CHILDREN is + * set. This flag handling is implemented in cgroup core for + * histrical reasons - the flag may be specified during mount. + * + * Currently, if any sibling cpusets have exclusive cpus or mem, we + * refuse to clone the configuration - thereby refusing the task to + * be entered, and as a result refusing the sys_unshare() or + * clone() which initiated it. If this becomes a problem for some + * users who wish to allow that scenario, then this could be + * changed to grant parent->cpus_allowed-sibling_cpus_exclusive + * (and likewise for mems) to the new cgroup. + */ + rcu_read_lock(); + cpuset_for_each_child(tmp_cs, pos_css, parent) { + if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs)) { + rcu_read_unlock(); + goto out_unlock; + } + } + rcu_read_unlock(); + + spin_lock_irq(&callback_lock); + cs->mems_allowed = parent->mems_allowed; + cs->effective_mems = parent->mems_allowed; + cpumask_copy(cs->cpus_allowed, parent->cpus_allowed); + cpumask_copy(cs->effective_cpus, parent->cpus_allowed); + spin_unlock_irq(&callback_lock); +out_unlock: + mutex_unlock(&cpuset_mutex); + return 0; +} + +/* + * If the cpuset being removed has its flag 'sched_load_balance' + * enabled, then simulate turning sched_load_balance off, which + * will call rebuild_sched_domains_locked(). + */ + +static void cpuset_css_offline(struct cgroup_subsys_state *css) +{ + struct cpuset *cs = css_cs(css); + + mutex_lock(&cpuset_mutex); + + if (is_sched_load_balance(cs)) + update_flag(CS_SCHED_LOAD_BALANCE, cs, 0); + + cpuset_dec(); + clear_bit(CS_ONLINE, &cs->flags); + + mutex_unlock(&cpuset_mutex); +} + +static void cpuset_css_free(struct cgroup_subsys_state *css) +{ + struct cpuset *cs = css_cs(css); + + free_cpumask_var(cs->effective_cpus); + free_cpumask_var(cs->cpus_allowed); + kfree(cs); +} + +static void cpuset_bind(struct cgroup_subsys_state *root_css) +{ + mutex_lock(&cpuset_mutex); + spin_lock_irq(&callback_lock); + + if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys)) { + cpumask_copy(top_cpuset.cpus_allowed, cpu_possible_mask); + top_cpuset.mems_allowed = node_possible_map; + } else { + cpumask_copy(top_cpuset.cpus_allowed, + top_cpuset.effective_cpus); + top_cpuset.mems_allowed = top_cpuset.effective_mems; + } + + spin_unlock_irq(&callback_lock); + mutex_unlock(&cpuset_mutex); +} + +/* + * Make sure the new task conform to the current state of its parent, + * which could have been changed by cpuset just after it inherits the + * state from the parent and before it sits on the cgroup's task list. + */ +static void cpuset_fork(struct task_struct *task) +{ + if (task_css_is_root(task, cpuset_cgrp_id)) + return; + + set_cpus_allowed_ptr(task, ¤t->cpus_allowed); + task->mems_allowed = current->mems_allowed; +} + +struct cgroup_subsys cpuset_cgrp_subsys = { + .css_alloc = cpuset_css_alloc, + .css_online = cpuset_css_online, + .css_offline = cpuset_css_offline, + .css_free = cpuset_css_free, + .can_attach = cpuset_can_attach, + .cancel_attach = cpuset_cancel_attach, + .attach = cpuset_attach, + .post_attach = cpuset_post_attach, + .bind = cpuset_bind, + .fork = cpuset_fork, + .legacy_cftypes = files, + .early_init = true, +}; + +/** + * cpuset_init - initialize cpusets at system boot + * + * Description: Initialize top_cpuset and the cpuset internal file system, + **/ + +int __init cpuset_init(void) +{ + int err = 0; + + if (!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL)) + BUG(); + if (!alloc_cpumask_var(&top_cpuset.effective_cpus, GFP_KERNEL)) + BUG(); + + cpumask_setall(top_cpuset.cpus_allowed); + nodes_setall(top_cpuset.mems_allowed); + cpumask_setall(top_cpuset.effective_cpus); + nodes_setall(top_cpuset.effective_mems); + + fmeter_init(&top_cpuset.fmeter); + set_bit(CS_SCHED_LOAD_BALANCE, &top_cpuset.flags); + top_cpuset.relax_domain_level = -1; + + err = register_filesystem(&cpuset_fs_type); + if (err < 0) + return err; + + if (!alloc_cpumask_var(&cpus_attach, GFP_KERNEL)) + BUG(); + + return 0; +} + +/* + * If CPU and/or memory hotplug handlers, below, unplug any CPUs + * or memory nodes, we need to walk over the cpuset hierarchy, + * removing that CPU or node from all cpusets. If this removes the + * last CPU or node from a cpuset, then move the tasks in the empty + * cpuset to its next-highest non-empty parent. + */ +static void remove_tasks_in_empty_cpuset(struct cpuset *cs) +{ + struct cpuset *parent; + + /* + * Find its next-highest non-empty parent, (top cpuset + * has online cpus, so can't be empty). + */ + parent = parent_cs(cs); + while (cpumask_empty(parent->cpus_allowed) || + nodes_empty(parent->mems_allowed)) + parent = parent_cs(parent); + + if (cgroup_transfer_tasks(parent->css.cgroup, cs->css.cgroup)) { + pr_err("cpuset: failed to transfer tasks out of empty cpuset "); + pr_cont_cgroup_name(cs->css.cgroup); + pr_cont("\n"); + } +} + +static void +hotplug_update_tasks_legacy(struct cpuset *cs, + struct cpumask *new_cpus, nodemask_t *new_mems, + bool cpus_updated, bool mems_updated) +{ + bool is_empty; + + spin_lock_irq(&callback_lock); + cpumask_copy(cs->cpus_allowed, new_cpus); + cpumask_copy(cs->effective_cpus, new_cpus); + cs->mems_allowed = *new_mems; + cs->effective_mems = *new_mems; + spin_unlock_irq(&callback_lock); + + /* + * Don't call update_tasks_cpumask() if the cpuset becomes empty, + * as the tasks will be migratecd to an ancestor. + */ + if (cpus_updated && !cpumask_empty(cs->cpus_allowed)) + update_tasks_cpumask(cs); + if (mems_updated && !nodes_empty(cs->mems_allowed)) + update_tasks_nodemask(cs); + + is_empty = cpumask_empty(cs->cpus_allowed) || + nodes_empty(cs->mems_allowed); + + mutex_unlock(&cpuset_mutex); + + /* + * Move tasks to the nearest ancestor with execution resources, + * This is full cgroup operation which will also call back into + * cpuset. Should be done outside any lock. + */ + if (is_empty) + remove_tasks_in_empty_cpuset(cs); + + mutex_lock(&cpuset_mutex); +} + +static void +hotplug_update_tasks(struct cpuset *cs, + struct cpumask *new_cpus, nodemask_t *new_mems, + bool cpus_updated, bool mems_updated) +{ + if (cpumask_empty(new_cpus)) + cpumask_copy(new_cpus, parent_cs(cs)->effective_cpus); + if (nodes_empty(*new_mems)) + *new_mems = parent_cs(cs)->effective_mems; + + spin_lock_irq(&callback_lock); + cpumask_copy(cs->effective_cpus, new_cpus); + cs->effective_mems = *new_mems; + spin_unlock_irq(&callback_lock); + + if (cpus_updated) + update_tasks_cpumask(cs); + if (mems_updated) + update_tasks_nodemask(cs); +} + +/** + * cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug + * @cs: cpuset in interest + * + * Compare @cs's cpu and mem masks against top_cpuset and if some have gone + * offline, update @cs accordingly. If @cs ends up with no CPU or memory, + * all its tasks are moved to the nearest ancestor with both resources. + */ +static void cpuset_hotplug_update_tasks(struct cpuset *cs) +{ + static cpumask_t new_cpus; + static nodemask_t new_mems; + bool cpus_updated; + bool mems_updated; +retry: + wait_event(cpuset_attach_wq, cs->attach_in_progress == 0); + + mutex_lock(&cpuset_mutex); + + /* + * We have raced with task attaching. We wait until attaching + * is finished, so we won't attach a task to an empty cpuset. + */ + if (cs->attach_in_progress) { + mutex_unlock(&cpuset_mutex); + goto retry; + } + + cpumask_and(&new_cpus, cs->cpus_allowed, parent_cs(cs)->effective_cpus); + nodes_and(new_mems, cs->mems_allowed, parent_cs(cs)->effective_mems); + + cpus_updated = !cpumask_equal(&new_cpus, cs->effective_cpus); + mems_updated = !nodes_equal(new_mems, cs->effective_mems); + + if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys)) + hotplug_update_tasks(cs, &new_cpus, &new_mems, + cpus_updated, mems_updated); + else + hotplug_update_tasks_legacy(cs, &new_cpus, &new_mems, + cpus_updated, mems_updated); + + mutex_unlock(&cpuset_mutex); +} + +/** + * cpuset_hotplug_workfn - handle CPU/memory hotunplug for a cpuset + * + * This function is called after either CPU or memory configuration has + * changed and updates cpuset accordingly. The top_cpuset is always + * synchronized to cpu_active_mask and N_MEMORY, which is necessary in + * order to make cpusets transparent (of no affect) on systems that are + * actively using CPU hotplug but making no active use of cpusets. + * + * Non-root cpusets are only affected by offlining. If any CPUs or memory + * nodes have been taken down, cpuset_hotplug_update_tasks() is invoked on + * all descendants. + * + * Note that CPU offlining during suspend is ignored. We don't modify + * cpusets across suspend/resume cycles at all. + */ +static void cpuset_hotplug_workfn(struct work_struct *work) +{ + static cpumask_t new_cpus; + static nodemask_t new_mems; + bool cpus_updated, mems_updated; + bool on_dfl = cgroup_subsys_on_dfl(cpuset_cgrp_subsys); + + mutex_lock(&cpuset_mutex); + + /* fetch the available cpus/mems and find out which changed how */ + cpumask_copy(&new_cpus, cpu_active_mask); + new_mems = node_states[N_MEMORY]; + + cpus_updated = !cpumask_equal(top_cpuset.effective_cpus, &new_cpus); + mems_updated = !nodes_equal(top_cpuset.effective_mems, new_mems); + + /* synchronize cpus_allowed to cpu_active_mask */ + if (cpus_updated) { + spin_lock_irq(&callback_lock); + if (!on_dfl) + cpumask_copy(top_cpuset.cpus_allowed, &new_cpus); + cpumask_copy(top_cpuset.effective_cpus, &new_cpus); + spin_unlock_irq(&callback_lock); + /* we don't mess with cpumasks of tasks in top_cpuset */ + } + + /* synchronize mems_allowed to N_MEMORY */ + if (mems_updated) { + spin_lock_irq(&callback_lock); + if (!on_dfl) + top_cpuset.mems_allowed = new_mems; + top_cpuset.effective_mems = new_mems; + spin_unlock_irq(&callback_lock); + update_tasks_nodemask(&top_cpuset); + } + + mutex_unlock(&cpuset_mutex); + + /* if cpus or mems changed, we need to propagate to descendants */ + if (cpus_updated || mems_updated) { + struct cpuset *cs; + struct cgroup_subsys_state *pos_css; + + rcu_read_lock(); + cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { + if (cs == &top_cpuset || !css_tryget_online(&cs->css)) + continue; + rcu_read_unlock(); + + cpuset_hotplug_update_tasks(cs); + + rcu_read_lock(); + css_put(&cs->css); + } + rcu_read_unlock(); + } + + /* rebuild sched domains if cpus_allowed has changed */ + if (cpus_updated) + rebuild_sched_domains(); +} + +void cpuset_update_active_cpus(bool cpu_online) +{ + /* + * We're inside cpu hotplug critical region which usually nests + * inside cgroup synchronization. Bounce actual hotplug processing + * to a work item to avoid reverse locking order. + * + * We still need to do partition_sched_domains() synchronously; + * otherwise, the scheduler will get confused and put tasks to the + * dead CPU. Fall back to the default single domain. + * cpuset_hotplug_workfn() will rebuild it as necessary. + */ + partition_sched_domains(1, NULL, NULL); + schedule_work(&cpuset_hotplug_work); +} + +/* + * Keep top_cpuset.mems_allowed tracking node_states[N_MEMORY]. + * Call this routine anytime after node_states[N_MEMORY] changes. + * See cpuset_update_active_cpus() for CPU hotplug handling. + */ +static int cpuset_track_online_nodes(struct notifier_block *self, + unsigned long action, void *arg) +{ + schedule_work(&cpuset_hotplug_work); + return NOTIFY_OK; +} + +static struct notifier_block cpuset_track_online_nodes_nb = { + .notifier_call = cpuset_track_online_nodes, + .priority = 10, /* ??! */ +}; + +/** + * cpuset_init_smp - initialize cpus_allowed + * + * Description: Finish top cpuset after cpu, node maps are initialized + */ +void __init cpuset_init_smp(void) +{ + cpumask_copy(top_cpuset.cpus_allowed, cpu_active_mask); + top_cpuset.mems_allowed = node_states[N_MEMORY]; + top_cpuset.old_mems_allowed = top_cpuset.mems_allowed; + + cpumask_copy(top_cpuset.effective_cpus, cpu_active_mask); + top_cpuset.effective_mems = node_states[N_MEMORY]; + + register_hotmemory_notifier(&cpuset_track_online_nodes_nb); + + cpuset_migrate_mm_wq = alloc_ordered_workqueue("cpuset_migrate_mm", 0); + BUG_ON(!cpuset_migrate_mm_wq); +} + +/** + * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset. + * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed. + * @pmask: pointer to struct cpumask variable to receive cpus_allowed set. + * + * Description: Returns the cpumask_var_t cpus_allowed of the cpuset + * attached to the specified @tsk. Guaranteed to return some non-empty + * subset of cpu_online_mask, even if this means going outside the + * tasks cpuset. + **/ + +void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask) +{ + unsigned long flags; + + spin_lock_irqsave(&callback_lock, flags); + rcu_read_lock(); + guarantee_online_cpus(task_cs(tsk), pmask); + rcu_read_unlock(); + spin_unlock_irqrestore(&callback_lock, flags); +} + +void cpuset_cpus_allowed_fallback(struct task_struct *tsk) +{ + rcu_read_lock(); + do_set_cpus_allowed(tsk, task_cs(tsk)->effective_cpus); + rcu_read_unlock(); + + /* + * We own tsk->cpus_allowed, nobody can change it under us. + * + * But we used cs && cs->cpus_allowed lockless and thus can + * race with cgroup_attach_task() or update_cpumask() and get + * the wrong tsk->cpus_allowed. However, both cases imply the + * subsequent cpuset_change_cpumask()->set_cpus_allowed_ptr() + * which takes task_rq_lock(). + * + * If we are called after it dropped the lock we must see all + * changes in tsk_cs()->cpus_allowed. Otherwise we can temporary + * set any mask even if it is not right from task_cs() pov, + * the pending set_cpus_allowed_ptr() will fix things. + * + * select_fallback_rq() will fix things ups and set cpu_possible_mask + * if required. + */ +} + +void __init cpuset_init_current_mems_allowed(void) +{ + nodes_setall(current->mems_allowed); +} + +/** + * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset. + * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed. + * + * Description: Returns the nodemask_t mems_allowed of the cpuset + * attached to the specified @tsk. Guaranteed to return some non-empty + * subset of node_states[N_MEMORY], even if this means going outside the + * tasks cpuset. + **/ + +nodemask_t cpuset_mems_allowed(struct task_struct *tsk) +{ + nodemask_t mask; + unsigned long flags; + + spin_lock_irqsave(&callback_lock, flags); + rcu_read_lock(); + guarantee_online_mems(task_cs(tsk), &mask); + rcu_read_unlock(); + spin_unlock_irqrestore(&callback_lock, flags); + + return mask; +} + +/** + * cpuset_nodemask_valid_mems_allowed - check nodemask vs. curremt mems_allowed + * @nodemask: the nodemask to be checked + * + * Are any of the nodes in the nodemask allowed in current->mems_allowed? + */ +int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) +{ + return nodes_intersects(*nodemask, current->mems_allowed); +} + +/* + * nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or + * mem_hardwall ancestor to the specified cpuset. Call holding + * callback_lock. If no ancestor is mem_exclusive or mem_hardwall + * (an unusual configuration), then returns the root cpuset. + */ +static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs) +{ + while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs)) + cs = parent_cs(cs); + return cs; +} + +/** + * cpuset_node_allowed - Can we allocate on a memory node? + * @node: is this an allowed node? + * @gfp_mask: memory allocation flags + * + * If we're in interrupt, yes, we can always allocate. If @node is set in + * current's mems_allowed, yes. If it's not a __GFP_HARDWALL request and this + * node is set in the nearest hardwalled cpuset ancestor to current's cpuset, + * yes. If current has access to memory reserves due to TIF_MEMDIE, yes. + * Otherwise, no. + * + * GFP_USER allocations are marked with the __GFP_HARDWALL bit, + * and do not allow allocations outside the current tasks cpuset + * unless the task has been OOM killed as is marked TIF_MEMDIE. + * GFP_KERNEL allocations are not so marked, so can escape to the + * nearest enclosing hardwalled ancestor cpuset. + * + * Scanning up parent cpusets requires callback_lock. The + * __alloc_pages() routine only calls here with __GFP_HARDWALL bit + * _not_ set if it's a GFP_KERNEL allocation, and all nodes in the + * current tasks mems_allowed came up empty on the first pass over + * the zonelist. So only GFP_KERNEL allocations, if all nodes in the + * cpuset are short of memory, might require taking the callback_lock. + * + * The first call here from mm/page_alloc:get_page_from_freelist() + * has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets, + * so no allocation on a node outside the cpuset is allowed (unless + * in interrupt, of course). + * + * The second pass through get_page_from_freelist() doesn't even call + * here for GFP_ATOMIC calls. For those calls, the __alloc_pages() + * variable 'wait' is not set, and the bit ALLOC_CPUSET is not set + * in alloc_flags. That logic and the checks below have the combined + * affect that: + * in_interrupt - any node ok (current task context irrelevant) + * GFP_ATOMIC - any node ok + * TIF_MEMDIE - any node ok + * GFP_KERNEL - any node in enclosing hardwalled cpuset ok + * GFP_USER - only nodes in current tasks mems allowed ok. + */ +bool __cpuset_node_allowed(int node, gfp_t gfp_mask) +{ + struct cpuset *cs; /* current cpuset ancestors */ + int allowed; /* is allocation in zone z allowed? */ + unsigned long flags; + + if (in_interrupt()) + return true; + if (node_isset(node, current->mems_allowed)) + return true; + /* + * Allow tasks that have access to memory reserves because they have + * been OOM killed to get memory anywhere. + */ + if (unlikely(test_thread_flag(TIF_MEMDIE))) + return true; + if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */ + return false; + + if (current->flags & PF_EXITING) /* Let dying task have memory */ + return true; + + /* Not hardwall and node outside mems_allowed: scan up cpusets */ + spin_lock_irqsave(&callback_lock, flags); + + rcu_read_lock(); + cs = nearest_hardwall_ancestor(task_cs(current)); + allowed = node_isset(node, cs->mems_allowed); + rcu_read_unlock(); + + spin_unlock_irqrestore(&callback_lock, flags); + return allowed; +} + +/** + * cpuset_mem_spread_node() - On which node to begin search for a file page + * cpuset_slab_spread_node() - On which node to begin search for a slab page + * + * If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for + * tasks in a cpuset with is_spread_page or is_spread_slab set), + * and if the memory allocation used cpuset_mem_spread_node() + * to determine on which node to start looking, as it will for + * certain page cache or slab cache pages such as used for file + * system buffers and inode caches, then instead of starting on the + * local node to look for a free page, rather spread the starting + * node around the tasks mems_allowed nodes. + * + * We don't have to worry about the returned node being offline + * because "it can't happen", and even if it did, it would be ok. + * + * The routines calling guarantee_online_mems() are careful to + * only set nodes in task->mems_allowed that are online. So it + * should not be possible for the following code to return an + * offline node. But if it did, that would be ok, as this routine + * is not returning the node where the allocation must be, only + * the node where the search should start. The zonelist passed to + * __alloc_pages() will include all nodes. If the slab allocator + * is passed an offline node, it will fall back to the local node. + * See kmem_cache_alloc_node(). + */ + +static int cpuset_spread_node(int *rotor) +{ + return *rotor = next_node_in(*rotor, current->mems_allowed); +} + +int cpuset_mem_spread_node(void) +{ + if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE) + current->cpuset_mem_spread_rotor = + node_random(¤t->mems_allowed); + + return cpuset_spread_node(¤t->cpuset_mem_spread_rotor); +} + +int cpuset_slab_spread_node(void) +{ + if (current->cpuset_slab_spread_rotor == NUMA_NO_NODE) + current->cpuset_slab_spread_rotor = + node_random(¤t->mems_allowed); + + return cpuset_spread_node(¤t->cpuset_slab_spread_rotor); +} + +EXPORT_SYMBOL_GPL(cpuset_mem_spread_node); + +/** + * cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's? + * @tsk1: pointer to task_struct of some task. + * @tsk2: pointer to task_struct of some other task. + * + * Description: Return true if @tsk1's mems_allowed intersects the + * mems_allowed of @tsk2. Used by the OOM killer to determine if + * one of the task's memory usage might impact the memory available + * to the other. + **/ + +int cpuset_mems_allowed_intersects(const struct task_struct *tsk1, + const struct task_struct *tsk2) +{ + return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed); +} + +/** + * cpuset_print_current_mems_allowed - prints current's cpuset and mems_allowed + * + * Description: Prints current's name, cpuset name, and cached copy of its + * mems_allowed to the kernel log. + */ +void cpuset_print_current_mems_allowed(void) +{ + struct cgroup *cgrp; + + rcu_read_lock(); + + cgrp = task_cs(current)->css.cgroup; + pr_info("%s cpuset=", current->comm); + pr_cont_cgroup_name(cgrp); + pr_cont(" mems_allowed=%*pbl\n", + nodemask_pr_args(¤t->mems_allowed)); + + rcu_read_unlock(); +} + +/* + * Collection of memory_pressure is suppressed unless + * this flag is enabled by writing "1" to the special + * cpuset file 'memory_pressure_enabled' in the root cpuset. + */ + +int cpuset_memory_pressure_enabled __read_mostly; + +/** + * cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims. + * + * Keep a running average of the rate of synchronous (direct) + * page reclaim efforts initiated by tasks in each cpuset. + * + * This represents the rate at which some task in the cpuset + * ran low on memory on all nodes it was allowed to use, and + * had to enter the kernels page reclaim code in an effort to + * create more free memory by tossing clean pages or swapping + * or writing dirty pages. + * + * Display to user space in the per-cpuset read-only file + * "memory_pressure". Value displayed is an integer + * representing the recent rate of entry into the synchronous + * (direct) page reclaim by any task attached to the cpuset. + **/ + +void __cpuset_memory_pressure_bump(void) +{ + rcu_read_lock(); + fmeter_markevent(&task_cs(current)->fmeter); + rcu_read_unlock(); +} + +#ifdef CONFIG_PROC_PID_CPUSET +/* + * proc_cpuset_show() + * - Print tasks cpuset path into seq_file. + * - Used for /proc//cpuset. + * - No need to task_lock(tsk) on this tsk->cpuset reference, as it + * doesn't really matter if tsk->cpuset changes after we read it, + * and we take cpuset_mutex, keeping cpuset_attach() from changing it + * anyway. + */ +int proc_cpuset_show(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *tsk) +{ + char *buf; + struct cgroup_subsys_state *css; + int retval; + + retval = -ENOMEM; + buf = kmalloc(PATH_MAX, GFP_KERNEL); + if (!buf) + goto out; + + css = task_get_css(tsk, cpuset_cgrp_id); + retval = cgroup_path_ns(css->cgroup, buf, PATH_MAX, + current->nsproxy->cgroup_ns); + css_put(css); + if (retval >= PATH_MAX) + retval = -ENAMETOOLONG; + if (retval < 0) + goto out_free; + seq_puts(m, buf); + seq_putc(m, '\n'); + retval = 0; +out_free: + kfree(buf); +out: + return retval; +} +#endif /* CONFIG_PROC_PID_CPUSET */ + +/* Display task mems_allowed in /proc//status file. */ +void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task) +{ + seq_printf(m, "Mems_allowed:\t%*pb\n", + nodemask_pr_args(&task->mems_allowed)); + seq_printf(m, "Mems_allowed_list:\t%*pbl\n", + nodemask_pr_args(&task->mems_allowed)); +} diff --git a/kernel/cgroup/freezer.c b/kernel/cgroup/freezer.c new file mode 100644 index 000000000000..1b72d56edce5 --- /dev/null +++ b/kernel/cgroup/freezer.c @@ -0,0 +1,481 @@ +/* + * cgroup_freezer.c - control group freezer subsystem + * + * Copyright IBM Corporation, 2007 + * + * Author : Cedric Le Goater + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of version 2.1 of the GNU Lesser General Public License + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it would be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * A cgroup is freezing if any FREEZING flags are set. FREEZING_SELF is + * set if "FROZEN" is written to freezer.state cgroupfs file, and cleared + * for "THAWED". FREEZING_PARENT is set if the parent freezer is FREEZING + * for whatever reason. IOW, a cgroup has FREEZING_PARENT set if one of + * its ancestors has FREEZING_SELF set. + */ +enum freezer_state_flags { + CGROUP_FREEZER_ONLINE = (1 << 0), /* freezer is fully online */ + CGROUP_FREEZING_SELF = (1 << 1), /* this freezer is freezing */ + CGROUP_FREEZING_PARENT = (1 << 2), /* the parent freezer is freezing */ + CGROUP_FROZEN = (1 << 3), /* this and its descendants frozen */ + + /* mask for all FREEZING flags */ + CGROUP_FREEZING = CGROUP_FREEZING_SELF | CGROUP_FREEZING_PARENT, +}; + +struct freezer { + struct cgroup_subsys_state css; + unsigned int state; +}; + +static DEFINE_MUTEX(freezer_mutex); + +static inline struct freezer *css_freezer(struct cgroup_subsys_state *css) +{ + return css ? container_of(css, struct freezer, css) : NULL; +} + +static inline struct freezer *task_freezer(struct task_struct *task) +{ + return css_freezer(task_css(task, freezer_cgrp_id)); +} + +static struct freezer *parent_freezer(struct freezer *freezer) +{ + return css_freezer(freezer->css.parent); +} + +bool cgroup_freezing(struct task_struct *task) +{ + bool ret; + + rcu_read_lock(); + ret = task_freezer(task)->state & CGROUP_FREEZING; + rcu_read_unlock(); + + return ret; +} + +static const char *freezer_state_strs(unsigned int state) +{ + if (state & CGROUP_FROZEN) + return "FROZEN"; + if (state & CGROUP_FREEZING) + return "FREEZING"; + return "THAWED"; +}; + +static struct cgroup_subsys_state * +freezer_css_alloc(struct cgroup_subsys_state *parent_css) +{ + struct freezer *freezer; + + freezer = kzalloc(sizeof(struct freezer), GFP_KERNEL); + if (!freezer) + return ERR_PTR(-ENOMEM); + + return &freezer->css; +} + +/** + * freezer_css_online - commit creation of a freezer css + * @css: css being created + * + * We're committing to creation of @css. Mark it online and inherit + * parent's freezing state while holding both parent's and our + * freezer->lock. + */ +static int freezer_css_online(struct cgroup_subsys_state *css) +{ + struct freezer *freezer = css_freezer(css); + struct freezer *parent = parent_freezer(freezer); + + mutex_lock(&freezer_mutex); + + freezer->state |= CGROUP_FREEZER_ONLINE; + + if (parent && (parent->state & CGROUP_FREEZING)) { + freezer->state |= CGROUP_FREEZING_PARENT | CGROUP_FROZEN; + atomic_inc(&system_freezing_cnt); + } + + mutex_unlock(&freezer_mutex); + return 0; +} + +/** + * freezer_css_offline - initiate destruction of a freezer css + * @css: css being destroyed + * + * @css is going away. Mark it dead and decrement system_freezing_count if + * it was holding one. + */ +static void freezer_css_offline(struct cgroup_subsys_state *css) +{ + struct freezer *freezer = css_freezer(css); + + mutex_lock(&freezer_mutex); + + if (freezer->state & CGROUP_FREEZING) + atomic_dec(&system_freezing_cnt); + + freezer->state = 0; + + mutex_unlock(&freezer_mutex); +} + +static void freezer_css_free(struct cgroup_subsys_state *css) +{ + kfree(css_freezer(css)); +} + +/* + * Tasks can be migrated into a different freezer anytime regardless of its + * current state. freezer_attach() is responsible for making new tasks + * conform to the current state. + * + * Freezer state changes and task migration are synchronized via + * @freezer->lock. freezer_attach() makes the new tasks conform to the + * current state and all following state changes can see the new tasks. + */ +static void freezer_attach(struct cgroup_taskset *tset) +{ + struct task_struct *task; + struct cgroup_subsys_state *new_css; + + mutex_lock(&freezer_mutex); + + /* + * Make the new tasks conform to the current state of @new_css. + * For simplicity, when migrating any task to a FROZEN cgroup, we + * revert it to FREEZING and let update_if_frozen() determine the + * correct state later. + * + * Tasks in @tset are on @new_css but may not conform to its + * current state before executing the following - !frozen tasks may + * be visible in a FROZEN cgroup and frozen tasks in a THAWED one. + */ + cgroup_taskset_for_each(task, new_css, tset) { + struct freezer *freezer = css_freezer(new_css); + + if (!(freezer->state & CGROUP_FREEZING)) { + __thaw_task(task); + } else { + freeze_task(task); + /* clear FROZEN and propagate upwards */ + while (freezer && (freezer->state & CGROUP_FROZEN)) { + freezer->state &= ~CGROUP_FROZEN; + freezer = parent_freezer(freezer); + } + } + } + + mutex_unlock(&freezer_mutex); +} + +/** + * freezer_fork - cgroup post fork callback + * @task: a task which has just been forked + * + * @task has just been created and should conform to the current state of + * the cgroup_freezer it belongs to. This function may race against + * freezer_attach(). Losing to freezer_attach() means that we don't have + * to do anything as freezer_attach() will put @task into the appropriate + * state. + */ +static void freezer_fork(struct task_struct *task) +{ + struct freezer *freezer; + + /* + * The root cgroup is non-freezable, so we can skip locking the + * freezer. This is safe regardless of race with task migration. + * If we didn't race or won, skipping is obviously the right thing + * to do. If we lost and root is the new cgroup, noop is still the + * right thing to do. + */ + if (task_css_is_root(task, freezer_cgrp_id)) + return; + + mutex_lock(&freezer_mutex); + rcu_read_lock(); + + freezer = task_freezer(task); + if (freezer->state & CGROUP_FREEZING) + freeze_task(task); + + rcu_read_unlock(); + mutex_unlock(&freezer_mutex); +} + +/** + * update_if_frozen - update whether a cgroup finished freezing + * @css: css of interest + * + * Once FREEZING is initiated, transition to FROZEN is lazily updated by + * calling this function. If the current state is FREEZING but not FROZEN, + * this function checks whether all tasks of this cgroup and the descendant + * cgroups finished freezing and, if so, sets FROZEN. + * + * The caller is responsible for grabbing RCU read lock and calling + * update_if_frozen() on all descendants prior to invoking this function. + * + * Task states and freezer state might disagree while tasks are being + * migrated into or out of @css, so we can't verify task states against + * @freezer state here. See freezer_attach() for details. + */ +static void update_if_frozen(struct cgroup_subsys_state *css) +{ + struct freezer *freezer = css_freezer(css); + struct cgroup_subsys_state *pos; + struct css_task_iter it; + struct task_struct *task; + + lockdep_assert_held(&freezer_mutex); + + if (!(freezer->state & CGROUP_FREEZING) || + (freezer->state & CGROUP_FROZEN)) + return; + + /* are all (live) children frozen? */ + rcu_read_lock(); + css_for_each_child(pos, css) { + struct freezer *child = css_freezer(pos); + + if ((child->state & CGROUP_FREEZER_ONLINE) && + !(child->state & CGROUP_FROZEN)) { + rcu_read_unlock(); + return; + } + } + rcu_read_unlock(); + + /* are all tasks frozen? */ + css_task_iter_start(css, &it); + + while ((task = css_task_iter_next(&it))) { + if (freezing(task)) { + /* + * freezer_should_skip() indicates that the task + * should be skipped when determining freezing + * completion. Consider it frozen in addition to + * the usual frozen condition. + */ + if (!frozen(task) && !freezer_should_skip(task)) + goto out_iter_end; + } + } + + freezer->state |= CGROUP_FROZEN; +out_iter_end: + css_task_iter_end(&it); +} + +static int freezer_read(struct seq_file *m, void *v) +{ + struct cgroup_subsys_state *css = seq_css(m), *pos; + + mutex_lock(&freezer_mutex); + rcu_read_lock(); + + /* update states bottom-up */ + css_for_each_descendant_post(pos, css) { + if (!css_tryget_online(pos)) + continue; + rcu_read_unlock(); + + update_if_frozen(pos); + + rcu_read_lock(); + css_put(pos); + } + + rcu_read_unlock(); + mutex_unlock(&freezer_mutex); + + seq_puts(m, freezer_state_strs(css_freezer(css)->state)); + seq_putc(m, '\n'); + return 0; +} + +static void freeze_cgroup(struct freezer *freezer) +{ + struct css_task_iter it; + struct task_struct *task; + + css_task_iter_start(&freezer->css, &it); + while ((task = css_task_iter_next(&it))) + freeze_task(task); + css_task_iter_end(&it); +} + +static void unfreeze_cgroup(struct freezer *freezer) +{ + struct css_task_iter it; + struct task_struct *task; + + css_task_iter_start(&freezer->css, &it); + while ((task = css_task_iter_next(&it))) + __thaw_task(task); + css_task_iter_end(&it); +} + +/** + * freezer_apply_state - apply state change to a single cgroup_freezer + * @freezer: freezer to apply state change to + * @freeze: whether to freeze or unfreeze + * @state: CGROUP_FREEZING_* flag to set or clear + * + * Set or clear @state on @cgroup according to @freeze, and perform + * freezing or thawing as necessary. + */ +static void freezer_apply_state(struct freezer *freezer, bool freeze, + unsigned int state) +{ + /* also synchronizes against task migration, see freezer_attach() */ + lockdep_assert_held(&freezer_mutex); + + if (!(freezer->state & CGROUP_FREEZER_ONLINE)) + return; + + if (freeze) { + if (!(freezer->state & CGROUP_FREEZING)) + atomic_inc(&system_freezing_cnt); + freezer->state |= state; + freeze_cgroup(freezer); + } else { + bool was_freezing = freezer->state & CGROUP_FREEZING; + + freezer->state &= ~state; + + if (!(freezer->state & CGROUP_FREEZING)) { + if (was_freezing) + atomic_dec(&system_freezing_cnt); + freezer->state &= ~CGROUP_FROZEN; + unfreeze_cgroup(freezer); + } + } +} + +/** + * freezer_change_state - change the freezing state of a cgroup_freezer + * @freezer: freezer of interest + * @freeze: whether to freeze or thaw + * + * Freeze or thaw @freezer according to @freeze. The operations are + * recursive - all descendants of @freezer will be affected. + */ +static void freezer_change_state(struct freezer *freezer, bool freeze) +{ + struct cgroup_subsys_state *pos; + + /* + * Update all its descendants in pre-order traversal. Each + * descendant will try to inherit its parent's FREEZING state as + * CGROUP_FREEZING_PARENT. + */ + mutex_lock(&freezer_mutex); + rcu_read_lock(); + css_for_each_descendant_pre(pos, &freezer->css) { + struct freezer *pos_f = css_freezer(pos); + struct freezer *parent = parent_freezer(pos_f); + + if (!css_tryget_online(pos)) + continue; + rcu_read_unlock(); + + if (pos_f == freezer) + freezer_apply_state(pos_f, freeze, + CGROUP_FREEZING_SELF); + else + freezer_apply_state(pos_f, + parent->state & CGROUP_FREEZING, + CGROUP_FREEZING_PARENT); + + rcu_read_lock(); + css_put(pos); + } + rcu_read_unlock(); + mutex_unlock(&freezer_mutex); +} + +static ssize_t freezer_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + bool freeze; + + buf = strstrip(buf); + + if (strcmp(buf, freezer_state_strs(0)) == 0) + freeze = false; + else if (strcmp(buf, freezer_state_strs(CGROUP_FROZEN)) == 0) + freeze = true; + else + return -EINVAL; + + freezer_change_state(css_freezer(of_css(of)), freeze); + return nbytes; +} + +static u64 freezer_self_freezing_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct freezer *freezer = css_freezer(css); + + return (bool)(freezer->state & CGROUP_FREEZING_SELF); +} + +static u64 freezer_parent_freezing_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct freezer *freezer = css_freezer(css); + + return (bool)(freezer->state & CGROUP_FREEZING_PARENT); +} + +static struct cftype files[] = { + { + .name = "state", + .flags = CFTYPE_NOT_ON_ROOT, + .seq_show = freezer_read, + .write = freezer_write, + }, + { + .name = "self_freezing", + .flags = CFTYPE_NOT_ON_ROOT, + .read_u64 = freezer_self_freezing_read, + }, + { + .name = "parent_freezing", + .flags = CFTYPE_NOT_ON_ROOT, + .read_u64 = freezer_parent_freezing_read, + }, + { } /* terminate */ +}; + +struct cgroup_subsys freezer_cgrp_subsys = { + .css_alloc = freezer_css_alloc, + .css_online = freezer_css_online, + .css_offline = freezer_css_offline, + .css_free = freezer_css_free, + .attach = freezer_attach, + .fork = freezer_fork, + .legacy_cftypes = files, +}; diff --git a/kernel/cgroup/pids.c b/kernel/cgroup/pids.c new file mode 100644 index 000000000000..2bd673783f1a --- /dev/null +++ b/kernel/cgroup/pids.c @@ -0,0 +1,348 @@ +/* + * Process number limiting controller for cgroups. + * + * Used to allow a cgroup hierarchy to stop any new processes from fork()ing + * after a certain limit is reached. + * + * Since it is trivial to hit the task limit without hitting any kmemcg limits + * in place, PIDs are a fundamental resource. As such, PID exhaustion must be + * preventable in the scope of a cgroup hierarchy by allowing resource limiting + * of the number of tasks in a cgroup. + * + * In order to use the `pids` controller, set the maximum number of tasks in + * pids.max (this is not available in the root cgroup for obvious reasons). The + * number of processes currently in the cgroup is given by pids.current. + * Organisational operations are not blocked by cgroup policies, so it is + * possible to have pids.current > pids.max. However, it is not possible to + * violate a cgroup policy through fork(). fork() will return -EAGAIN if forking + * would cause a cgroup policy to be violated. + * + * To set a cgroup to have no limit, set pids.max to "max". This is the default + * for all new cgroups (N.B. that PID limits are hierarchical, so the most + * stringent limit in the hierarchy is followed). + * + * pids.current tracks all child cgroup hierarchies, so parent/pids.current is + * a superset of parent/child/pids.current. + * + * Copyright (C) 2015 Aleksa Sarai + * + * This file is subject to the terms and conditions of version 2 of the GNU + * General Public License. See the file COPYING in the main directory of the + * Linux distribution for more details. + */ + +#include +#include +#include +#include +#include + +#define PIDS_MAX (PID_MAX_LIMIT + 1ULL) +#define PIDS_MAX_STR "max" + +struct pids_cgroup { + struct cgroup_subsys_state css; + + /* + * Use 64-bit types so that we can safely represent "max" as + * %PIDS_MAX = (%PID_MAX_LIMIT + 1). + */ + atomic64_t counter; + int64_t limit; + + /* Handle for "pids.events" */ + struct cgroup_file events_file; + + /* Number of times fork failed because limit was hit. */ + atomic64_t events_limit; +}; + +static struct pids_cgroup *css_pids(struct cgroup_subsys_state *css) +{ + return container_of(css, struct pids_cgroup, css); +} + +static struct pids_cgroup *parent_pids(struct pids_cgroup *pids) +{ + return css_pids(pids->css.parent); +} + +static struct cgroup_subsys_state * +pids_css_alloc(struct cgroup_subsys_state *parent) +{ + struct pids_cgroup *pids; + + pids = kzalloc(sizeof(struct pids_cgroup), GFP_KERNEL); + if (!pids) + return ERR_PTR(-ENOMEM); + + pids->limit = PIDS_MAX; + atomic64_set(&pids->counter, 0); + atomic64_set(&pids->events_limit, 0); + return &pids->css; +} + +static void pids_css_free(struct cgroup_subsys_state *css) +{ + kfree(css_pids(css)); +} + +/** + * pids_cancel - uncharge the local pid count + * @pids: the pid cgroup state + * @num: the number of pids to cancel + * + * This function will WARN if the pid count goes under 0, because such a case is + * a bug in the pids controller proper. + */ +static void pids_cancel(struct pids_cgroup *pids, int num) +{ + /* + * A negative count (or overflow for that matter) is invalid, + * and indicates a bug in the `pids` controller proper. + */ + WARN_ON_ONCE(atomic64_add_negative(-num, &pids->counter)); +} + +/** + * pids_uncharge - hierarchically uncharge the pid count + * @pids: the pid cgroup state + * @num: the number of pids to uncharge + */ +static void pids_uncharge(struct pids_cgroup *pids, int num) +{ + struct pids_cgroup *p; + + for (p = pids; parent_pids(p); p = parent_pids(p)) + pids_cancel(p, num); +} + +/** + * pids_charge - hierarchically charge the pid count + * @pids: the pid cgroup state + * @num: the number of pids to charge + * + * This function does *not* follow the pid limit set. It cannot fail and the new + * pid count may exceed the limit. This is only used for reverting failed + * attaches, where there is no other way out than violating the limit. + */ +static void pids_charge(struct pids_cgroup *pids, int num) +{ + struct pids_cgroup *p; + + for (p = pids; parent_pids(p); p = parent_pids(p)) + atomic64_add(num, &p->counter); +} + +/** + * pids_try_charge - hierarchically try to charge the pid count + * @pids: the pid cgroup state + * @num: the number of pids to charge + * + * This function follows the set limit. It will fail if the charge would cause + * the new value to exceed the hierarchical limit. Returns 0 if the charge + * succeeded, otherwise -EAGAIN. + */ +static int pids_try_charge(struct pids_cgroup *pids, int num) +{ + struct pids_cgroup *p, *q; + + for (p = pids; parent_pids(p); p = parent_pids(p)) { + int64_t new = atomic64_add_return(num, &p->counter); + + /* + * Since new is capped to the maximum number of pid_t, if + * p->limit is %PIDS_MAX then we know that this test will never + * fail. + */ + if (new > p->limit) + goto revert; + } + + return 0; + +revert: + for (q = pids; q != p; q = parent_pids(q)) + pids_cancel(q, num); + pids_cancel(p, num); + + return -EAGAIN; +} + +static int pids_can_attach(struct cgroup_taskset *tset) +{ + struct task_struct *task; + struct cgroup_subsys_state *dst_css; + + cgroup_taskset_for_each(task, dst_css, tset) { + struct pids_cgroup *pids = css_pids(dst_css); + struct cgroup_subsys_state *old_css; + struct pids_cgroup *old_pids; + + /* + * No need to pin @old_css between here and cancel_attach() + * because cgroup core protects it from being freed before + * the migration completes or fails. + */ + old_css = task_css(task, pids_cgrp_id); + old_pids = css_pids(old_css); + + pids_charge(pids, 1); + pids_uncharge(old_pids, 1); + } + + return 0; +} + +static void pids_cancel_attach(struct cgroup_taskset *tset) +{ + struct task_struct *task; + struct cgroup_subsys_state *dst_css; + + cgroup_taskset_for_each(task, dst_css, tset) { + struct pids_cgroup *pids = css_pids(dst_css); + struct cgroup_subsys_state *old_css; + struct pids_cgroup *old_pids; + + old_css = task_css(task, pids_cgrp_id); + old_pids = css_pids(old_css); + + pids_charge(old_pids, 1); + pids_uncharge(pids, 1); + } +} + +/* + * task_css_check(true) in pids_can_fork() and pids_cancel_fork() relies + * on threadgroup_change_begin() held by the copy_process(). + */ +static int pids_can_fork(struct task_struct *task) +{ + struct cgroup_subsys_state *css; + struct pids_cgroup *pids; + int err; + + css = task_css_check(current, pids_cgrp_id, true); + pids = css_pids(css); + err = pids_try_charge(pids, 1); + if (err) { + /* Only log the first time events_limit is incremented. */ + if (atomic64_inc_return(&pids->events_limit) == 1) { + pr_info("cgroup: fork rejected by pids controller in "); + pr_cont_cgroup_path(task_cgroup(current, pids_cgrp_id)); + pr_cont("\n"); + } + cgroup_file_notify(&pids->events_file); + } + return err; +} + +static void pids_cancel_fork(struct task_struct *task) +{ + struct cgroup_subsys_state *css; + struct pids_cgroup *pids; + + css = task_css_check(current, pids_cgrp_id, true); + pids = css_pids(css); + pids_uncharge(pids, 1); +} + +static void pids_free(struct task_struct *task) +{ + struct pids_cgroup *pids = css_pids(task_css(task, pids_cgrp_id)); + + pids_uncharge(pids, 1); +} + +static ssize_t pids_max_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off) +{ + struct cgroup_subsys_state *css = of_css(of); + struct pids_cgroup *pids = css_pids(css); + int64_t limit; + int err; + + buf = strstrip(buf); + if (!strcmp(buf, PIDS_MAX_STR)) { + limit = PIDS_MAX; + goto set_limit; + } + + err = kstrtoll(buf, 0, &limit); + if (err) + return err; + + if (limit < 0 || limit >= PIDS_MAX) + return -EINVAL; + +set_limit: + /* + * Limit updates don't need to be mutex'd, since it isn't + * critical that any racing fork()s follow the new limit. + */ + pids->limit = limit; + return nbytes; +} + +static int pids_max_show(struct seq_file *sf, void *v) +{ + struct cgroup_subsys_state *css = seq_css(sf); + struct pids_cgroup *pids = css_pids(css); + int64_t limit = pids->limit; + + if (limit >= PIDS_MAX) + seq_printf(sf, "%s\n", PIDS_MAX_STR); + else + seq_printf(sf, "%lld\n", limit); + + return 0; +} + +static s64 pids_current_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct pids_cgroup *pids = css_pids(css); + + return atomic64_read(&pids->counter); +} + +static int pids_events_show(struct seq_file *sf, void *v) +{ + struct pids_cgroup *pids = css_pids(seq_css(sf)); + + seq_printf(sf, "max %lld\n", (s64)atomic64_read(&pids->events_limit)); + return 0; +} + +static struct cftype pids_files[] = { + { + .name = "max", + .write = pids_max_write, + .seq_show = pids_max_show, + .flags = CFTYPE_NOT_ON_ROOT, + }, + { + .name = "current", + .read_s64 = pids_current_read, + .flags = CFTYPE_NOT_ON_ROOT, + }, + { + .name = "events", + .seq_show = pids_events_show, + .file_offset = offsetof(struct pids_cgroup, events_file), + .flags = CFTYPE_NOT_ON_ROOT, + }, + { } /* terminate */ +}; + +struct cgroup_subsys pids_cgrp_subsys = { + .css_alloc = pids_css_alloc, + .css_free = pids_css_free, + .can_attach = pids_can_attach, + .cancel_attach = pids_cancel_attach, + .can_fork = pids_can_fork, + .cancel_fork = pids_cancel_fork, + .free = pids_free, + .legacy_cftypes = pids_files, + .dfl_cftypes = pids_files, +}; diff --git a/kernel/cgroup_freezer.c b/kernel/cgroup_freezer.c deleted file mode 100644 index 1b72d56edce5..000000000000 --- a/kernel/cgroup_freezer.c +++ /dev/null @@ -1,481 +0,0 @@ -/* - * cgroup_freezer.c - control group freezer subsystem - * - * Copyright IBM Corporation, 2007 - * - * Author : Cedric Le Goater - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of version 2.1 of the GNU Lesser General Public License - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it would be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * A cgroup is freezing if any FREEZING flags are set. FREEZING_SELF is - * set if "FROZEN" is written to freezer.state cgroupfs file, and cleared - * for "THAWED". FREEZING_PARENT is set if the parent freezer is FREEZING - * for whatever reason. IOW, a cgroup has FREEZING_PARENT set if one of - * its ancestors has FREEZING_SELF set. - */ -enum freezer_state_flags { - CGROUP_FREEZER_ONLINE = (1 << 0), /* freezer is fully online */ - CGROUP_FREEZING_SELF = (1 << 1), /* this freezer is freezing */ - CGROUP_FREEZING_PARENT = (1 << 2), /* the parent freezer is freezing */ - CGROUP_FROZEN = (1 << 3), /* this and its descendants frozen */ - - /* mask for all FREEZING flags */ - CGROUP_FREEZING = CGROUP_FREEZING_SELF | CGROUP_FREEZING_PARENT, -}; - -struct freezer { - struct cgroup_subsys_state css; - unsigned int state; -}; - -static DEFINE_MUTEX(freezer_mutex); - -static inline struct freezer *css_freezer(struct cgroup_subsys_state *css) -{ - return css ? container_of(css, struct freezer, css) : NULL; -} - -static inline struct freezer *task_freezer(struct task_struct *task) -{ - return css_freezer(task_css(task, freezer_cgrp_id)); -} - -static struct freezer *parent_freezer(struct freezer *freezer) -{ - return css_freezer(freezer->css.parent); -} - -bool cgroup_freezing(struct task_struct *task) -{ - bool ret; - - rcu_read_lock(); - ret = task_freezer(task)->state & CGROUP_FREEZING; - rcu_read_unlock(); - - return ret; -} - -static const char *freezer_state_strs(unsigned int state) -{ - if (state & CGROUP_FROZEN) - return "FROZEN"; - if (state & CGROUP_FREEZING) - return "FREEZING"; - return "THAWED"; -}; - -static struct cgroup_subsys_state * -freezer_css_alloc(struct cgroup_subsys_state *parent_css) -{ - struct freezer *freezer; - - freezer = kzalloc(sizeof(struct freezer), GFP_KERNEL); - if (!freezer) - return ERR_PTR(-ENOMEM); - - return &freezer->css; -} - -/** - * freezer_css_online - commit creation of a freezer css - * @css: css being created - * - * We're committing to creation of @css. Mark it online and inherit - * parent's freezing state while holding both parent's and our - * freezer->lock. - */ -static int freezer_css_online(struct cgroup_subsys_state *css) -{ - struct freezer *freezer = css_freezer(css); - struct freezer *parent = parent_freezer(freezer); - - mutex_lock(&freezer_mutex); - - freezer->state |= CGROUP_FREEZER_ONLINE; - - if (parent && (parent->state & CGROUP_FREEZING)) { - freezer->state |= CGROUP_FREEZING_PARENT | CGROUP_FROZEN; - atomic_inc(&system_freezing_cnt); - } - - mutex_unlock(&freezer_mutex); - return 0; -} - -/** - * freezer_css_offline - initiate destruction of a freezer css - * @css: css being destroyed - * - * @css is going away. Mark it dead and decrement system_freezing_count if - * it was holding one. - */ -static void freezer_css_offline(struct cgroup_subsys_state *css) -{ - struct freezer *freezer = css_freezer(css); - - mutex_lock(&freezer_mutex); - - if (freezer->state & CGROUP_FREEZING) - atomic_dec(&system_freezing_cnt); - - freezer->state = 0; - - mutex_unlock(&freezer_mutex); -} - -static void freezer_css_free(struct cgroup_subsys_state *css) -{ - kfree(css_freezer(css)); -} - -/* - * Tasks can be migrated into a different freezer anytime regardless of its - * current state. freezer_attach() is responsible for making new tasks - * conform to the current state. - * - * Freezer state changes and task migration are synchronized via - * @freezer->lock. freezer_attach() makes the new tasks conform to the - * current state and all following state changes can see the new tasks. - */ -static void freezer_attach(struct cgroup_taskset *tset) -{ - struct task_struct *task; - struct cgroup_subsys_state *new_css; - - mutex_lock(&freezer_mutex); - - /* - * Make the new tasks conform to the current state of @new_css. - * For simplicity, when migrating any task to a FROZEN cgroup, we - * revert it to FREEZING and let update_if_frozen() determine the - * correct state later. - * - * Tasks in @tset are on @new_css but may not conform to its - * current state before executing the following - !frozen tasks may - * be visible in a FROZEN cgroup and frozen tasks in a THAWED one. - */ - cgroup_taskset_for_each(task, new_css, tset) { - struct freezer *freezer = css_freezer(new_css); - - if (!(freezer->state & CGROUP_FREEZING)) { - __thaw_task(task); - } else { - freeze_task(task); - /* clear FROZEN and propagate upwards */ - while (freezer && (freezer->state & CGROUP_FROZEN)) { - freezer->state &= ~CGROUP_FROZEN; - freezer = parent_freezer(freezer); - } - } - } - - mutex_unlock(&freezer_mutex); -} - -/** - * freezer_fork - cgroup post fork callback - * @task: a task which has just been forked - * - * @task has just been created and should conform to the current state of - * the cgroup_freezer it belongs to. This function may race against - * freezer_attach(). Losing to freezer_attach() means that we don't have - * to do anything as freezer_attach() will put @task into the appropriate - * state. - */ -static void freezer_fork(struct task_struct *task) -{ - struct freezer *freezer; - - /* - * The root cgroup is non-freezable, so we can skip locking the - * freezer. This is safe regardless of race with task migration. - * If we didn't race or won, skipping is obviously the right thing - * to do. If we lost and root is the new cgroup, noop is still the - * right thing to do. - */ - if (task_css_is_root(task, freezer_cgrp_id)) - return; - - mutex_lock(&freezer_mutex); - rcu_read_lock(); - - freezer = task_freezer(task); - if (freezer->state & CGROUP_FREEZING) - freeze_task(task); - - rcu_read_unlock(); - mutex_unlock(&freezer_mutex); -} - -/** - * update_if_frozen - update whether a cgroup finished freezing - * @css: css of interest - * - * Once FREEZING is initiated, transition to FROZEN is lazily updated by - * calling this function. If the current state is FREEZING but not FROZEN, - * this function checks whether all tasks of this cgroup and the descendant - * cgroups finished freezing and, if so, sets FROZEN. - * - * The caller is responsible for grabbing RCU read lock and calling - * update_if_frozen() on all descendants prior to invoking this function. - * - * Task states and freezer state might disagree while tasks are being - * migrated into or out of @css, so we can't verify task states against - * @freezer state here. See freezer_attach() for details. - */ -static void update_if_frozen(struct cgroup_subsys_state *css) -{ - struct freezer *freezer = css_freezer(css); - struct cgroup_subsys_state *pos; - struct css_task_iter it; - struct task_struct *task; - - lockdep_assert_held(&freezer_mutex); - - if (!(freezer->state & CGROUP_FREEZING) || - (freezer->state & CGROUP_FROZEN)) - return; - - /* are all (live) children frozen? */ - rcu_read_lock(); - css_for_each_child(pos, css) { - struct freezer *child = css_freezer(pos); - - if ((child->state & CGROUP_FREEZER_ONLINE) && - !(child->state & CGROUP_FROZEN)) { - rcu_read_unlock(); - return; - } - } - rcu_read_unlock(); - - /* are all tasks frozen? */ - css_task_iter_start(css, &it); - - while ((task = css_task_iter_next(&it))) { - if (freezing(task)) { - /* - * freezer_should_skip() indicates that the task - * should be skipped when determining freezing - * completion. Consider it frozen in addition to - * the usual frozen condition. - */ - if (!frozen(task) && !freezer_should_skip(task)) - goto out_iter_end; - } - } - - freezer->state |= CGROUP_FROZEN; -out_iter_end: - css_task_iter_end(&it); -} - -static int freezer_read(struct seq_file *m, void *v) -{ - struct cgroup_subsys_state *css = seq_css(m), *pos; - - mutex_lock(&freezer_mutex); - rcu_read_lock(); - - /* update states bottom-up */ - css_for_each_descendant_post(pos, css) { - if (!css_tryget_online(pos)) - continue; - rcu_read_unlock(); - - update_if_frozen(pos); - - rcu_read_lock(); - css_put(pos); - } - - rcu_read_unlock(); - mutex_unlock(&freezer_mutex); - - seq_puts(m, freezer_state_strs(css_freezer(css)->state)); - seq_putc(m, '\n'); - return 0; -} - -static void freeze_cgroup(struct freezer *freezer) -{ - struct css_task_iter it; - struct task_struct *task; - - css_task_iter_start(&freezer->css, &it); - while ((task = css_task_iter_next(&it))) - freeze_task(task); - css_task_iter_end(&it); -} - -static void unfreeze_cgroup(struct freezer *freezer) -{ - struct css_task_iter it; - struct task_struct *task; - - css_task_iter_start(&freezer->css, &it); - while ((task = css_task_iter_next(&it))) - __thaw_task(task); - css_task_iter_end(&it); -} - -/** - * freezer_apply_state - apply state change to a single cgroup_freezer - * @freezer: freezer to apply state change to - * @freeze: whether to freeze or unfreeze - * @state: CGROUP_FREEZING_* flag to set or clear - * - * Set or clear @state on @cgroup according to @freeze, and perform - * freezing or thawing as necessary. - */ -static void freezer_apply_state(struct freezer *freezer, bool freeze, - unsigned int state) -{ - /* also synchronizes against task migration, see freezer_attach() */ - lockdep_assert_held(&freezer_mutex); - - if (!(freezer->state & CGROUP_FREEZER_ONLINE)) - return; - - if (freeze) { - if (!(freezer->state & CGROUP_FREEZING)) - atomic_inc(&system_freezing_cnt); - freezer->state |= state; - freeze_cgroup(freezer); - } else { - bool was_freezing = freezer->state & CGROUP_FREEZING; - - freezer->state &= ~state; - - if (!(freezer->state & CGROUP_FREEZING)) { - if (was_freezing) - atomic_dec(&system_freezing_cnt); - freezer->state &= ~CGROUP_FROZEN; - unfreeze_cgroup(freezer); - } - } -} - -/** - * freezer_change_state - change the freezing state of a cgroup_freezer - * @freezer: freezer of interest - * @freeze: whether to freeze or thaw - * - * Freeze or thaw @freezer according to @freeze. The operations are - * recursive - all descendants of @freezer will be affected. - */ -static void freezer_change_state(struct freezer *freezer, bool freeze) -{ - struct cgroup_subsys_state *pos; - - /* - * Update all its descendants in pre-order traversal. Each - * descendant will try to inherit its parent's FREEZING state as - * CGROUP_FREEZING_PARENT. - */ - mutex_lock(&freezer_mutex); - rcu_read_lock(); - css_for_each_descendant_pre(pos, &freezer->css) { - struct freezer *pos_f = css_freezer(pos); - struct freezer *parent = parent_freezer(pos_f); - - if (!css_tryget_online(pos)) - continue; - rcu_read_unlock(); - - if (pos_f == freezer) - freezer_apply_state(pos_f, freeze, - CGROUP_FREEZING_SELF); - else - freezer_apply_state(pos_f, - parent->state & CGROUP_FREEZING, - CGROUP_FREEZING_PARENT); - - rcu_read_lock(); - css_put(pos); - } - rcu_read_unlock(); - mutex_unlock(&freezer_mutex); -} - -static ssize_t freezer_write(struct kernfs_open_file *of, - char *buf, size_t nbytes, loff_t off) -{ - bool freeze; - - buf = strstrip(buf); - - if (strcmp(buf, freezer_state_strs(0)) == 0) - freeze = false; - else if (strcmp(buf, freezer_state_strs(CGROUP_FROZEN)) == 0) - freeze = true; - else - return -EINVAL; - - freezer_change_state(css_freezer(of_css(of)), freeze); - return nbytes; -} - -static u64 freezer_self_freezing_read(struct cgroup_subsys_state *css, - struct cftype *cft) -{ - struct freezer *freezer = css_freezer(css); - - return (bool)(freezer->state & CGROUP_FREEZING_SELF); -} - -static u64 freezer_parent_freezing_read(struct cgroup_subsys_state *css, - struct cftype *cft) -{ - struct freezer *freezer = css_freezer(css); - - return (bool)(freezer->state & CGROUP_FREEZING_PARENT); -} - -static struct cftype files[] = { - { - .name = "state", - .flags = CFTYPE_NOT_ON_ROOT, - .seq_show = freezer_read, - .write = freezer_write, - }, - { - .name = "self_freezing", - .flags = CFTYPE_NOT_ON_ROOT, - .read_u64 = freezer_self_freezing_read, - }, - { - .name = "parent_freezing", - .flags = CFTYPE_NOT_ON_ROOT, - .read_u64 = freezer_parent_freezing_read, - }, - { } /* terminate */ -}; - -struct cgroup_subsys freezer_cgrp_subsys = { - .css_alloc = freezer_css_alloc, - .css_online = freezer_css_online, - .css_offline = freezer_css_offline, - .css_free = freezer_css_free, - .attach = freezer_attach, - .fork = freezer_fork, - .legacy_cftypes = files, -}; diff --git a/kernel/cgroup_pids.c b/kernel/cgroup_pids.c deleted file mode 100644 index 2bd673783f1a..000000000000 --- a/kernel/cgroup_pids.c +++ /dev/null @@ -1,348 +0,0 @@ -/* - * Process number limiting controller for cgroups. - * - * Used to allow a cgroup hierarchy to stop any new processes from fork()ing - * after a certain limit is reached. - * - * Since it is trivial to hit the task limit without hitting any kmemcg limits - * in place, PIDs are a fundamental resource. As such, PID exhaustion must be - * preventable in the scope of a cgroup hierarchy by allowing resource limiting - * of the number of tasks in a cgroup. - * - * In order to use the `pids` controller, set the maximum number of tasks in - * pids.max (this is not available in the root cgroup for obvious reasons). The - * number of processes currently in the cgroup is given by pids.current. - * Organisational operations are not blocked by cgroup policies, so it is - * possible to have pids.current > pids.max. However, it is not possible to - * violate a cgroup policy through fork(). fork() will return -EAGAIN if forking - * would cause a cgroup policy to be violated. - * - * To set a cgroup to have no limit, set pids.max to "max". This is the default - * for all new cgroups (N.B. that PID limits are hierarchical, so the most - * stringent limit in the hierarchy is followed). - * - * pids.current tracks all child cgroup hierarchies, so parent/pids.current is - * a superset of parent/child/pids.current. - * - * Copyright (C) 2015 Aleksa Sarai - * - * This file is subject to the terms and conditions of version 2 of the GNU - * General Public License. See the file COPYING in the main directory of the - * Linux distribution for more details. - */ - -#include -#include -#include -#include -#include - -#define PIDS_MAX (PID_MAX_LIMIT + 1ULL) -#define PIDS_MAX_STR "max" - -struct pids_cgroup { - struct cgroup_subsys_state css; - - /* - * Use 64-bit types so that we can safely represent "max" as - * %PIDS_MAX = (%PID_MAX_LIMIT + 1). - */ - atomic64_t counter; - int64_t limit; - - /* Handle for "pids.events" */ - struct cgroup_file events_file; - - /* Number of times fork failed because limit was hit. */ - atomic64_t events_limit; -}; - -static struct pids_cgroup *css_pids(struct cgroup_subsys_state *css) -{ - return container_of(css, struct pids_cgroup, css); -} - -static struct pids_cgroup *parent_pids(struct pids_cgroup *pids) -{ - return css_pids(pids->css.parent); -} - -static struct cgroup_subsys_state * -pids_css_alloc(struct cgroup_subsys_state *parent) -{ - struct pids_cgroup *pids; - - pids = kzalloc(sizeof(struct pids_cgroup), GFP_KERNEL); - if (!pids) - return ERR_PTR(-ENOMEM); - - pids->limit = PIDS_MAX; - atomic64_set(&pids->counter, 0); - atomic64_set(&pids->events_limit, 0); - return &pids->css; -} - -static void pids_css_free(struct cgroup_subsys_state *css) -{ - kfree(css_pids(css)); -} - -/** - * pids_cancel - uncharge the local pid count - * @pids: the pid cgroup state - * @num: the number of pids to cancel - * - * This function will WARN if the pid count goes under 0, because such a case is - * a bug in the pids controller proper. - */ -static void pids_cancel(struct pids_cgroup *pids, int num) -{ - /* - * A negative count (or overflow for that matter) is invalid, - * and indicates a bug in the `pids` controller proper. - */ - WARN_ON_ONCE(atomic64_add_negative(-num, &pids->counter)); -} - -/** - * pids_uncharge - hierarchically uncharge the pid count - * @pids: the pid cgroup state - * @num: the number of pids to uncharge - */ -static void pids_uncharge(struct pids_cgroup *pids, int num) -{ - struct pids_cgroup *p; - - for (p = pids; parent_pids(p); p = parent_pids(p)) - pids_cancel(p, num); -} - -/** - * pids_charge - hierarchically charge the pid count - * @pids: the pid cgroup state - * @num: the number of pids to charge - * - * This function does *not* follow the pid limit set. It cannot fail and the new - * pid count may exceed the limit. This is only used for reverting failed - * attaches, where there is no other way out than violating the limit. - */ -static void pids_charge(struct pids_cgroup *pids, int num) -{ - struct pids_cgroup *p; - - for (p = pids; parent_pids(p); p = parent_pids(p)) - atomic64_add(num, &p->counter); -} - -/** - * pids_try_charge - hierarchically try to charge the pid count - * @pids: the pid cgroup state - * @num: the number of pids to charge - * - * This function follows the set limit. It will fail if the charge would cause - * the new value to exceed the hierarchical limit. Returns 0 if the charge - * succeeded, otherwise -EAGAIN. - */ -static int pids_try_charge(struct pids_cgroup *pids, int num) -{ - struct pids_cgroup *p, *q; - - for (p = pids; parent_pids(p); p = parent_pids(p)) { - int64_t new = atomic64_add_return(num, &p->counter); - - /* - * Since new is capped to the maximum number of pid_t, if - * p->limit is %PIDS_MAX then we know that this test will never - * fail. - */ - if (new > p->limit) - goto revert; - } - - return 0; - -revert: - for (q = pids; q != p; q = parent_pids(q)) - pids_cancel(q, num); - pids_cancel(p, num); - - return -EAGAIN; -} - -static int pids_can_attach(struct cgroup_taskset *tset) -{ - struct task_struct *task; - struct cgroup_subsys_state *dst_css; - - cgroup_taskset_for_each(task, dst_css, tset) { - struct pids_cgroup *pids = css_pids(dst_css); - struct cgroup_subsys_state *old_css; - struct pids_cgroup *old_pids; - - /* - * No need to pin @old_css between here and cancel_attach() - * because cgroup core protects it from being freed before - * the migration completes or fails. - */ - old_css = task_css(task, pids_cgrp_id); - old_pids = css_pids(old_css); - - pids_charge(pids, 1); - pids_uncharge(old_pids, 1); - } - - return 0; -} - -static void pids_cancel_attach(struct cgroup_taskset *tset) -{ - struct task_struct *task; - struct cgroup_subsys_state *dst_css; - - cgroup_taskset_for_each(task, dst_css, tset) { - struct pids_cgroup *pids = css_pids(dst_css); - struct cgroup_subsys_state *old_css; - struct pids_cgroup *old_pids; - - old_css = task_css(task, pids_cgrp_id); - old_pids = css_pids(old_css); - - pids_charge(old_pids, 1); - pids_uncharge(pids, 1); - } -} - -/* - * task_css_check(true) in pids_can_fork() and pids_cancel_fork() relies - * on threadgroup_change_begin() held by the copy_process(). - */ -static int pids_can_fork(struct task_struct *task) -{ - struct cgroup_subsys_state *css; - struct pids_cgroup *pids; - int err; - - css = task_css_check(current, pids_cgrp_id, true); - pids = css_pids(css); - err = pids_try_charge(pids, 1); - if (err) { - /* Only log the first time events_limit is incremented. */ - if (atomic64_inc_return(&pids->events_limit) == 1) { - pr_info("cgroup: fork rejected by pids controller in "); - pr_cont_cgroup_path(task_cgroup(current, pids_cgrp_id)); - pr_cont("\n"); - } - cgroup_file_notify(&pids->events_file); - } - return err; -} - -static void pids_cancel_fork(struct task_struct *task) -{ - struct cgroup_subsys_state *css; - struct pids_cgroup *pids; - - css = task_css_check(current, pids_cgrp_id, true); - pids = css_pids(css); - pids_uncharge(pids, 1); -} - -static void pids_free(struct task_struct *task) -{ - struct pids_cgroup *pids = css_pids(task_css(task, pids_cgrp_id)); - - pids_uncharge(pids, 1); -} - -static ssize_t pids_max_write(struct kernfs_open_file *of, char *buf, - size_t nbytes, loff_t off) -{ - struct cgroup_subsys_state *css = of_css(of); - struct pids_cgroup *pids = css_pids(css); - int64_t limit; - int err; - - buf = strstrip(buf); - if (!strcmp(buf, PIDS_MAX_STR)) { - limit = PIDS_MAX; - goto set_limit; - } - - err = kstrtoll(buf, 0, &limit); - if (err) - return err; - - if (limit < 0 || limit >= PIDS_MAX) - return -EINVAL; - -set_limit: - /* - * Limit updates don't need to be mutex'd, since it isn't - * critical that any racing fork()s follow the new limit. - */ - pids->limit = limit; - return nbytes; -} - -static int pids_max_show(struct seq_file *sf, void *v) -{ - struct cgroup_subsys_state *css = seq_css(sf); - struct pids_cgroup *pids = css_pids(css); - int64_t limit = pids->limit; - - if (limit >= PIDS_MAX) - seq_printf(sf, "%s\n", PIDS_MAX_STR); - else - seq_printf(sf, "%lld\n", limit); - - return 0; -} - -static s64 pids_current_read(struct cgroup_subsys_state *css, - struct cftype *cft) -{ - struct pids_cgroup *pids = css_pids(css); - - return atomic64_read(&pids->counter); -} - -static int pids_events_show(struct seq_file *sf, void *v) -{ - struct pids_cgroup *pids = css_pids(seq_css(sf)); - - seq_printf(sf, "max %lld\n", (s64)atomic64_read(&pids->events_limit)); - return 0; -} - -static struct cftype pids_files[] = { - { - .name = "max", - .write = pids_max_write, - .seq_show = pids_max_show, - .flags = CFTYPE_NOT_ON_ROOT, - }, - { - .name = "current", - .read_s64 = pids_current_read, - .flags = CFTYPE_NOT_ON_ROOT, - }, - { - .name = "events", - .seq_show = pids_events_show, - .file_offset = offsetof(struct pids_cgroup, events_file), - .flags = CFTYPE_NOT_ON_ROOT, - }, - { } /* terminate */ -}; - -struct cgroup_subsys pids_cgrp_subsys = { - .css_alloc = pids_css_alloc, - .css_free = pids_css_free, - .can_attach = pids_can_attach, - .cancel_attach = pids_cancel_attach, - .can_fork = pids_can_fork, - .cancel_fork = pids_cancel_fork, - .free = pids_free, - .legacy_cftypes = pids_files, - .dfl_cftypes = pids_files, -}; diff --git a/kernel/cpuset.c b/kernel/cpuset.c deleted file mode 100644 index b3088886cd37..000000000000 --- a/kernel/cpuset.c +++ /dev/null @@ -1,2752 +0,0 @@ -/* - * kernel/cpuset.c - * - * Processor and Memory placement constraints for sets of tasks. - * - * Copyright (C) 2003 BULL SA. - * Copyright (C) 2004-2007 Silicon Graphics, Inc. - * Copyright (C) 2006 Google, Inc - * - * Portions derived from Patrick Mochel's sysfs code. - * sysfs is Copyright (c) 2001-3 Patrick Mochel - * - * 2003-10-10 Written by Simon Derr. - * 2003-10-22 Updates by Stephen Hemminger. - * 2004 May-July Rework by Paul Jackson. - * 2006 Rework by Paul Menage to use generic cgroups - * 2008 Rework of the scheduler domains and CPU hotplug handling - * by Max Krasnyansky - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file COPYING in the main directory of the Linux - * distribution for more details. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key); - -/* See "Frequency meter" comments, below. */ - -struct fmeter { - int cnt; /* unprocessed events count */ - int val; /* most recent output value */ - time64_t time; /* clock (secs) when val computed */ - spinlock_t lock; /* guards read or write of above */ -}; - -struct cpuset { - struct cgroup_subsys_state css; - - unsigned long flags; /* "unsigned long" so bitops work */ - - /* - * On default hierarchy: - * - * The user-configured masks can only be changed by writing to - * cpuset.cpus and cpuset.mems, and won't be limited by the - * parent masks. - * - * The effective masks is the real masks that apply to the tasks - * in the cpuset. They may be changed if the configured masks are - * changed or hotplug happens. - * - * effective_mask == configured_mask & parent's effective_mask, - * and if it ends up empty, it will inherit the parent's mask. - * - * - * On legacy hierachy: - * - * The user-configured masks are always the same with effective masks. - */ - - /* user-configured CPUs and Memory Nodes allow to tasks */ - cpumask_var_t cpus_allowed; - nodemask_t mems_allowed; - - /* effective CPUs and Memory Nodes allow to tasks */ - cpumask_var_t effective_cpus; - nodemask_t effective_mems; - - /* - * This is old Memory Nodes tasks took on. - * - * - top_cpuset.old_mems_allowed is initialized to mems_allowed. - * - A new cpuset's old_mems_allowed is initialized when some - * task is moved into it. - * - old_mems_allowed is used in cpuset_migrate_mm() when we change - * cpuset.mems_allowed and have tasks' nodemask updated, and - * then old_mems_allowed is updated to mems_allowed. - */ - nodemask_t old_mems_allowed; - - struct fmeter fmeter; /* memory_pressure filter */ - - /* - * Tasks are being attached to this cpuset. Used to prevent - * zeroing cpus/mems_allowed between ->can_attach() and ->attach(). - */ - int attach_in_progress; - - /* partition number for rebuild_sched_domains() */ - int pn; - - /* for custom sched domain */ - int relax_domain_level; -}; - -static inline struct cpuset *css_cs(struct cgroup_subsys_state *css) -{ - return css ? container_of(css, struct cpuset, css) : NULL; -} - -/* Retrieve the cpuset for a task */ -static inline struct cpuset *task_cs(struct task_struct *task) -{ - return css_cs(task_css(task, cpuset_cgrp_id)); -} - -static inline struct cpuset *parent_cs(struct cpuset *cs) -{ - return css_cs(cs->css.parent); -} - -#ifdef CONFIG_NUMA -static inline bool task_has_mempolicy(struct task_struct *task) -{ - return task->mempolicy; -} -#else -static inline bool task_has_mempolicy(struct task_struct *task) -{ - return false; -} -#endif - - -/* bits in struct cpuset flags field */ -typedef enum { - CS_ONLINE, - CS_CPU_EXCLUSIVE, - CS_MEM_EXCLUSIVE, - CS_MEM_HARDWALL, - CS_MEMORY_MIGRATE, - CS_SCHED_LOAD_BALANCE, - CS_SPREAD_PAGE, - CS_SPREAD_SLAB, -} cpuset_flagbits_t; - -/* convenient tests for these bits */ -static inline bool is_cpuset_online(const struct cpuset *cs) -{ - return test_bit(CS_ONLINE, &cs->flags); -} - -static inline int is_cpu_exclusive(const struct cpuset *cs) -{ - return test_bit(CS_CPU_EXCLUSIVE, &cs->flags); -} - -static inline int is_mem_exclusive(const struct cpuset *cs) -{ - return test_bit(CS_MEM_EXCLUSIVE, &cs->flags); -} - -static inline int is_mem_hardwall(const struct cpuset *cs) -{ - return test_bit(CS_MEM_HARDWALL, &cs->flags); -} - -static inline int is_sched_load_balance(const struct cpuset *cs) -{ - return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); -} - -static inline int is_memory_migrate(const struct cpuset *cs) -{ - return test_bit(CS_MEMORY_MIGRATE, &cs->flags); -} - -static inline int is_spread_page(const struct cpuset *cs) -{ - return test_bit(CS_SPREAD_PAGE, &cs->flags); -} - -static inline int is_spread_slab(const struct cpuset *cs) -{ - return test_bit(CS_SPREAD_SLAB, &cs->flags); -} - -static struct cpuset top_cpuset = { - .flags = ((1 << CS_ONLINE) | (1 << CS_CPU_EXCLUSIVE) | - (1 << CS_MEM_EXCLUSIVE)), -}; - -/** - * cpuset_for_each_child - traverse online children of a cpuset - * @child_cs: loop cursor pointing to the current child - * @pos_css: used for iteration - * @parent_cs: target cpuset to walk children of - * - * Walk @child_cs through the online children of @parent_cs. Must be used - * with RCU read locked. - */ -#define cpuset_for_each_child(child_cs, pos_css, parent_cs) \ - css_for_each_child((pos_css), &(parent_cs)->css) \ - if (is_cpuset_online(((child_cs) = css_cs((pos_css))))) - -/** - * cpuset_for_each_descendant_pre - pre-order walk of a cpuset's descendants - * @des_cs: loop cursor pointing to the current descendant - * @pos_css: used for iteration - * @root_cs: target cpuset to walk ancestor of - * - * Walk @des_cs through the online descendants of @root_cs. Must be used - * with RCU read locked. The caller may modify @pos_css by calling - * css_rightmost_descendant() to skip subtree. @root_cs is included in the - * iteration and the first node to be visited. - */ -#define cpuset_for_each_descendant_pre(des_cs, pos_css, root_cs) \ - css_for_each_descendant_pre((pos_css), &(root_cs)->css) \ - if (is_cpuset_online(((des_cs) = css_cs((pos_css))))) - -/* - * There are two global locks guarding cpuset structures - cpuset_mutex and - * callback_lock. We also require taking task_lock() when dereferencing a - * task's cpuset pointer. See "The task_lock() exception", at the end of this - * comment. - * - * A task must hold both locks to modify cpusets. If a task holds - * cpuset_mutex, then it blocks others wanting that mutex, ensuring that it - * is the only task able to also acquire callback_lock and be able to - * modify cpusets. It can perform various checks on the cpuset structure - * first, knowing nothing will change. It can also allocate memory while - * just holding cpuset_mutex. While it is performing these checks, various - * callback routines can briefly acquire callback_lock to query cpusets. - * Once it is ready to make the changes, it takes callback_lock, blocking - * everyone else. - * - * Calls to the kernel memory allocator can not be made while holding - * callback_lock, as that would risk double tripping on callback_lock - * from one of the callbacks into the cpuset code from within - * __alloc_pages(). - * - * If a task is only holding callback_lock, then it has read-only - * access to cpusets. - * - * Now, the task_struct fields mems_allowed and mempolicy may be changed - * by other task, we use alloc_lock in the task_struct fields to protect - * them. - * - * The cpuset_common_file_read() handlers only hold callback_lock across - * small pieces of code, such as when reading out possibly multi-word - * cpumasks and nodemasks. - * - * Accessing a task's cpuset should be done in accordance with the - * guidelines for accessing subsystem state in kernel/cgroup.c - */ - -static DEFINE_MUTEX(cpuset_mutex); -static DEFINE_SPINLOCK(callback_lock); - -static struct workqueue_struct *cpuset_migrate_mm_wq; - -/* - * CPU / memory hotplug is handled asynchronously. - */ -static void cpuset_hotplug_workfn(struct work_struct *work); -static DECLARE_WORK(cpuset_hotplug_work, cpuset_hotplug_workfn); - -static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq); - -/* - * This is ugly, but preserves the userspace API for existing cpuset - * users. If someone tries to mount the "cpuset" filesystem, we - * silently switch it to mount "cgroup" instead - */ -static struct dentry *cpuset_mount(struct file_system_type *fs_type, - int flags, const char *unused_dev_name, void *data) -{ - struct file_system_type *cgroup_fs = get_fs_type("cgroup"); - struct dentry *ret = ERR_PTR(-ENODEV); - if (cgroup_fs) { - char mountopts[] = - "cpuset,noprefix," - "release_agent=/sbin/cpuset_release_agent"; - ret = cgroup_fs->mount(cgroup_fs, flags, - unused_dev_name, mountopts); - put_filesystem(cgroup_fs); - } - return ret; -} - -static struct file_system_type cpuset_fs_type = { - .name = "cpuset", - .mount = cpuset_mount, -}; - -/* - * Return in pmask the portion of a cpusets's cpus_allowed that - * are online. If none are online, walk up the cpuset hierarchy - * until we find one that does have some online cpus. - * - * One way or another, we guarantee to return some non-empty subset - * of cpu_online_mask. - * - * Call with callback_lock or cpuset_mutex held. - */ -static void guarantee_online_cpus(struct cpuset *cs, struct cpumask *pmask) -{ - while (!cpumask_intersects(cs->effective_cpus, cpu_online_mask)) { - cs = parent_cs(cs); - if (unlikely(!cs)) { - /* - * The top cpuset doesn't have any online cpu as a - * consequence of a race between cpuset_hotplug_work - * and cpu hotplug notifier. But we know the top - * cpuset's effective_cpus is on its way to to be - * identical to cpu_online_mask. - */ - cpumask_copy(pmask, cpu_online_mask); - return; - } - } - cpumask_and(pmask, cs->effective_cpus, cpu_online_mask); -} - -/* - * Return in *pmask the portion of a cpusets's mems_allowed that - * are online, with memory. If none are online with memory, walk - * up the cpuset hierarchy until we find one that does have some - * online mems. The top cpuset always has some mems online. - * - * One way or another, we guarantee to return some non-empty subset - * of node_states[N_MEMORY]. - * - * Call with callback_lock or cpuset_mutex held. - */ -static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask) -{ - while (!nodes_intersects(cs->effective_mems, node_states[N_MEMORY])) - cs = parent_cs(cs); - nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]); -} - -/* - * update task's spread flag if cpuset's page/slab spread flag is set - * - * Call with callback_lock or cpuset_mutex held. - */ -static void cpuset_update_task_spread_flag(struct cpuset *cs, - struct task_struct *tsk) -{ - if (is_spread_page(cs)) - task_set_spread_page(tsk); - else - task_clear_spread_page(tsk); - - if (is_spread_slab(cs)) - task_set_spread_slab(tsk); - else - task_clear_spread_slab(tsk); -} - -/* - * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q? - * - * One cpuset is a subset of another if all its allowed CPUs and - * Memory Nodes are a subset of the other, and its exclusive flags - * are only set if the other's are set. Call holding cpuset_mutex. - */ - -static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q) -{ - return cpumask_subset(p->cpus_allowed, q->cpus_allowed) && - nodes_subset(p->mems_allowed, q->mems_allowed) && - is_cpu_exclusive(p) <= is_cpu_exclusive(q) && - is_mem_exclusive(p) <= is_mem_exclusive(q); -} - -/** - * alloc_trial_cpuset - allocate a trial cpuset - * @cs: the cpuset that the trial cpuset duplicates - */ -static struct cpuset *alloc_trial_cpuset(struct cpuset *cs) -{ - struct cpuset *trial; - - trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL); - if (!trial) - return NULL; - - if (!alloc_cpumask_var(&trial->cpus_allowed, GFP_KERNEL)) - goto free_cs; - if (!alloc_cpumask_var(&trial->effective_cpus, GFP_KERNEL)) - goto free_cpus; - - cpumask_copy(trial->cpus_allowed, cs->cpus_allowed); - cpumask_copy(trial->effective_cpus, cs->effective_cpus); - return trial; - -free_cpus: - free_cpumask_var(trial->cpus_allowed); -free_cs: - kfree(trial); - return NULL; -} - -/** - * free_trial_cpuset - free the trial cpuset - * @trial: the trial cpuset to be freed - */ -static void free_trial_cpuset(struct cpuset *trial) -{ - free_cpumask_var(trial->effective_cpus); - free_cpumask_var(trial->cpus_allowed); - kfree(trial); -} - -/* - * validate_change() - Used to validate that any proposed cpuset change - * follows the structural rules for cpusets. - * - * If we replaced the flag and mask values of the current cpuset - * (cur) with those values in the trial cpuset (trial), would - * our various subset and exclusive rules still be valid? Presumes - * cpuset_mutex held. - * - * 'cur' is the address of an actual, in-use cpuset. Operations - * such as list traversal that depend on the actual address of the - * cpuset in the list must use cur below, not trial. - * - * 'trial' is the address of bulk structure copy of cur, with - * perhaps one or more of the fields cpus_allowed, mems_allowed, - * or flags changed to new, trial values. - * - * Return 0 if valid, -errno if not. - */ - -static int validate_change(struct cpuset *cur, struct cpuset *trial) -{ - struct cgroup_subsys_state *css; - struct cpuset *c, *par; - int ret; - - rcu_read_lock(); - - /* Each of our child cpusets must be a subset of us */ - ret = -EBUSY; - cpuset_for_each_child(c, css, cur) - if (!is_cpuset_subset(c, trial)) - goto out; - - /* Remaining checks don't apply to root cpuset */ - ret = 0; - if (cur == &top_cpuset) - goto out; - - par = parent_cs(cur); - - /* On legacy hiearchy, we must be a subset of our parent cpuset. */ - ret = -EACCES; - if (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && - !is_cpuset_subset(trial, par)) - goto out; - - /* - * If either I or some sibling (!= me) is exclusive, we can't - * overlap - */ - ret = -EINVAL; - cpuset_for_each_child(c, css, par) { - if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) && - c != cur && - cpumask_intersects(trial->cpus_allowed, c->cpus_allowed)) - goto out; - if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) && - c != cur && - nodes_intersects(trial->mems_allowed, c->mems_allowed)) - goto out; - } - - /* - * Cpusets with tasks - existing or newly being attached - can't - * be changed to have empty cpus_allowed or mems_allowed. - */ - ret = -ENOSPC; - if ((cgroup_is_populated(cur->css.cgroup) || cur->attach_in_progress)) { - if (!cpumask_empty(cur->cpus_allowed) && - cpumask_empty(trial->cpus_allowed)) - goto out; - if (!nodes_empty(cur->mems_allowed) && - nodes_empty(trial->mems_allowed)) - goto out; - } - - /* - * We can't shrink if we won't have enough room for SCHED_DEADLINE - * tasks. - */ - ret = -EBUSY; - if (is_cpu_exclusive(cur) && - !cpuset_cpumask_can_shrink(cur->cpus_allowed, - trial->cpus_allowed)) - goto out; - - ret = 0; -out: - rcu_read_unlock(); - return ret; -} - -#ifdef CONFIG_SMP -/* - * Helper routine for generate_sched_domains(). - * Do cpusets a, b have overlapping effective cpus_allowed masks? - */ -static int cpusets_overlap(struct cpuset *a, struct cpuset *b) -{ - return cpumask_intersects(a->effective_cpus, b->effective_cpus); -} - -static void -update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c) -{ - if (dattr->relax_domain_level < c->relax_domain_level) - dattr->relax_domain_level = c->relax_domain_level; - return; -} - -static void update_domain_attr_tree(struct sched_domain_attr *dattr, - struct cpuset *root_cs) -{ - struct cpuset *cp; - struct cgroup_subsys_state *pos_css; - - rcu_read_lock(); - cpuset_for_each_descendant_pre(cp, pos_css, root_cs) { - /* skip the whole subtree if @cp doesn't have any CPU */ - if (cpumask_empty(cp->cpus_allowed)) { - pos_css = css_rightmost_descendant(pos_css); - continue; - } - - if (is_sched_load_balance(cp)) - update_domain_attr(dattr, cp); - } - rcu_read_unlock(); -} - -/* - * generate_sched_domains() - * - * This function builds a partial partition of the systems CPUs - * A 'partial partition' is a set of non-overlapping subsets whose - * union is a subset of that set. - * The output of this function needs to be passed to kernel/sched/core.c - * partition_sched_domains() routine, which will rebuild the scheduler's - * load balancing domains (sched domains) as specified by that partial - * partition. - * - * See "What is sched_load_balance" in Documentation/cgroups/cpusets.txt - * for a background explanation of this. - * - * Does not return errors, on the theory that the callers of this - * routine would rather not worry about failures to rebuild sched - * domains when operating in the severe memory shortage situations - * that could cause allocation failures below. - * - * Must be called with cpuset_mutex held. - * - * The three key local variables below are: - * q - a linked-list queue of cpuset pointers, used to implement a - * top-down scan of all cpusets. This scan loads a pointer - * to each cpuset marked is_sched_load_balance into the - * array 'csa'. For our purposes, rebuilding the schedulers - * sched domains, we can ignore !is_sched_load_balance cpusets. - * csa - (for CpuSet Array) Array of pointers to all the cpusets - * that need to be load balanced, for convenient iterative - * access by the subsequent code that finds the best partition, - * i.e the set of domains (subsets) of CPUs such that the - * cpus_allowed of every cpuset marked is_sched_load_balance - * is a subset of one of these domains, while there are as - * many such domains as possible, each as small as possible. - * doms - Conversion of 'csa' to an array of cpumasks, for passing to - * the kernel/sched/core.c routine partition_sched_domains() in a - * convenient format, that can be easily compared to the prior - * value to determine what partition elements (sched domains) - * were changed (added or removed.) - * - * Finding the best partition (set of domains): - * The triple nested loops below over i, j, k scan over the - * load balanced cpusets (using the array of cpuset pointers in - * csa[]) looking for pairs of cpusets that have overlapping - * cpus_allowed, but which don't have the same 'pn' partition - * number and gives them in the same partition number. It keeps - * looping on the 'restart' label until it can no longer find - * any such pairs. - * - * The union of the cpus_allowed masks from the set of - * all cpusets having the same 'pn' value then form the one - * element of the partition (one sched domain) to be passed to - * partition_sched_domains(). - */ -static int generate_sched_domains(cpumask_var_t **domains, - struct sched_domain_attr **attributes) -{ - struct cpuset *cp; /* scans q */ - struct cpuset **csa; /* array of all cpuset ptrs */ - int csn; /* how many cpuset ptrs in csa so far */ - int i, j, k; /* indices for partition finding loops */ - cpumask_var_t *doms; /* resulting partition; i.e. sched domains */ - cpumask_var_t non_isolated_cpus; /* load balanced CPUs */ - struct sched_domain_attr *dattr; /* attributes for custom domains */ - int ndoms = 0; /* number of sched domains in result */ - int nslot; /* next empty doms[] struct cpumask slot */ - struct cgroup_subsys_state *pos_css; - - doms = NULL; - dattr = NULL; - csa = NULL; - - if (!alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL)) - goto done; - cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map); - - /* Special case for the 99% of systems with one, full, sched domain */ - if (is_sched_load_balance(&top_cpuset)) { - ndoms = 1; - doms = alloc_sched_domains(ndoms); - if (!doms) - goto done; - - dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL); - if (dattr) { - *dattr = SD_ATTR_INIT; - update_domain_attr_tree(dattr, &top_cpuset); - } - cpumask_and(doms[0], top_cpuset.effective_cpus, - non_isolated_cpus); - - goto done; - } - - csa = kmalloc(nr_cpusets() * sizeof(cp), GFP_KERNEL); - if (!csa) - goto done; - csn = 0; - - rcu_read_lock(); - cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) { - if (cp == &top_cpuset) - continue; - /* - * Continue traversing beyond @cp iff @cp has some CPUs and - * isn't load balancing. The former is obvious. The - * latter: All child cpusets contain a subset of the - * parent's cpus, so just skip them, and then we call - * update_domain_attr_tree() to calc relax_domain_level of - * the corresponding sched domain. - */ - if (!cpumask_empty(cp->cpus_allowed) && - !(is_sched_load_balance(cp) && - cpumask_intersects(cp->cpus_allowed, non_isolated_cpus))) - continue; - - if (is_sched_load_balance(cp)) - csa[csn++] = cp; - - /* skip @cp's subtree */ - pos_css = css_rightmost_descendant(pos_css); - } - rcu_read_unlock(); - - for (i = 0; i < csn; i++) - csa[i]->pn = i; - ndoms = csn; - -restart: - /* Find the best partition (set of sched domains) */ - for (i = 0; i < csn; i++) { - struct cpuset *a = csa[i]; - int apn = a->pn; - - for (j = 0; j < csn; j++) { - struct cpuset *b = csa[j]; - int bpn = b->pn; - - if (apn != bpn && cpusets_overlap(a, b)) { - for (k = 0; k < csn; k++) { - struct cpuset *c = csa[k]; - - if (c->pn == bpn) - c->pn = apn; - } - ndoms--; /* one less element */ - goto restart; - } - } - } - - /* - * Now we know how many domains to create. - * Convert to and populate cpu masks. - */ - doms = alloc_sched_domains(ndoms); - if (!doms) - goto done; - - /* - * The rest of the code, including the scheduler, can deal with - * dattr==NULL case. No need to abort if alloc fails. - */ - dattr = kmalloc(ndoms * sizeof(struct sched_domain_attr), GFP_KERNEL); - - for (nslot = 0, i = 0; i < csn; i++) { - struct cpuset *a = csa[i]; - struct cpumask *dp; - int apn = a->pn; - - if (apn < 0) { - /* Skip completed partitions */ - continue; - } - - dp = doms[nslot]; - - if (nslot == ndoms) { - static int warnings = 10; - if (warnings) { - pr_warn("rebuild_sched_domains confused: nslot %d, ndoms %d, csn %d, i %d, apn %d\n", - nslot, ndoms, csn, i, apn); - warnings--; - } - continue; - } - - cpumask_clear(dp); - if (dattr) - *(dattr + nslot) = SD_ATTR_INIT; - for (j = i; j < csn; j++) { - struct cpuset *b = csa[j]; - - if (apn == b->pn) { - cpumask_or(dp, dp, b->effective_cpus); - cpumask_and(dp, dp, non_isolated_cpus); - if (dattr) - update_domain_attr_tree(dattr + nslot, b); - - /* Done with this partition */ - b->pn = -1; - } - } - nslot++; - } - BUG_ON(nslot != ndoms); - -done: - free_cpumask_var(non_isolated_cpus); - kfree(csa); - - /* - * Fallback to the default domain if kmalloc() failed. - * See comments in partition_sched_domains(). - */ - if (doms == NULL) - ndoms = 1; - - *domains = doms; - *attributes = dattr; - return ndoms; -} - -/* - * Rebuild scheduler domains. - * - * If the flag 'sched_load_balance' of any cpuset with non-empty - * 'cpus' changes, or if the 'cpus' allowed changes in any cpuset - * which has that flag enabled, or if any cpuset with a non-empty - * 'cpus' is removed, then call this routine to rebuild the - * scheduler's dynamic sched domains. - * - * Call with cpuset_mutex held. Takes get_online_cpus(). - */ -static void rebuild_sched_domains_locked(void) -{ - struct sched_domain_attr *attr; - cpumask_var_t *doms; - int ndoms; - - lockdep_assert_held(&cpuset_mutex); - get_online_cpus(); - - /* - * We have raced with CPU hotplug. Don't do anything to avoid - * passing doms with offlined cpu to partition_sched_domains(). - * Anyways, hotplug work item will rebuild sched domains. - */ - if (!cpumask_equal(top_cpuset.effective_cpus, cpu_active_mask)) - goto out; - - /* Generate domain masks and attrs */ - ndoms = generate_sched_domains(&doms, &attr); - - /* Have scheduler rebuild the domains */ - partition_sched_domains(ndoms, doms, attr); -out: - put_online_cpus(); -} -#else /* !CONFIG_SMP */ -static void rebuild_sched_domains_locked(void) -{ -} -#endif /* CONFIG_SMP */ - -void rebuild_sched_domains(void) -{ - mutex_lock(&cpuset_mutex); - rebuild_sched_domains_locked(); - mutex_unlock(&cpuset_mutex); -} - -/** - * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset. - * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed - * - * Iterate through each task of @cs updating its cpus_allowed to the - * effective cpuset's. As this function is called with cpuset_mutex held, - * cpuset membership stays stable. - */ -static void update_tasks_cpumask(struct cpuset *cs) -{ - struct css_task_iter it; - struct task_struct *task; - - css_task_iter_start(&cs->css, &it); - while ((task = css_task_iter_next(&it))) - set_cpus_allowed_ptr(task, cs->effective_cpus); - css_task_iter_end(&it); -} - -/* - * update_cpumasks_hier - Update effective cpumasks and tasks in the subtree - * @cs: the cpuset to consider - * @new_cpus: temp variable for calculating new effective_cpus - * - * When congifured cpumask is changed, the effective cpumasks of this cpuset - * and all its descendants need to be updated. - * - * On legacy hierachy, effective_cpus will be the same with cpu_allowed. - * - * Called with cpuset_mutex held - */ -static void update_cpumasks_hier(struct cpuset *cs, struct cpumask *new_cpus) -{ - struct cpuset *cp; - struct cgroup_subsys_state *pos_css; - bool need_rebuild_sched_domains = false; - - rcu_read_lock(); - cpuset_for_each_descendant_pre(cp, pos_css, cs) { - struct cpuset *parent = parent_cs(cp); - - cpumask_and(new_cpus, cp->cpus_allowed, parent->effective_cpus); - - /* - * If it becomes empty, inherit the effective mask of the - * parent, which is guaranteed to have some CPUs. - */ - if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && - cpumask_empty(new_cpus)) - cpumask_copy(new_cpus, parent->effective_cpus); - - /* Skip the whole subtree if the cpumask remains the same. */ - if (cpumask_equal(new_cpus, cp->effective_cpus)) { - pos_css = css_rightmost_descendant(pos_css); - continue; - } - - if (!css_tryget_online(&cp->css)) - continue; - rcu_read_unlock(); - - spin_lock_irq(&callback_lock); - cpumask_copy(cp->effective_cpus, new_cpus); - spin_unlock_irq(&callback_lock); - - WARN_ON(!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && - !cpumask_equal(cp->cpus_allowed, cp->effective_cpus)); - - update_tasks_cpumask(cp); - - /* - * If the effective cpumask of any non-empty cpuset is changed, - * we need to rebuild sched domains. - */ - if (!cpumask_empty(cp->cpus_allowed) && - is_sched_load_balance(cp)) - need_rebuild_sched_domains = true; - - rcu_read_lock(); - css_put(&cp->css); - } - rcu_read_unlock(); - - if (need_rebuild_sched_domains) - rebuild_sched_domains_locked(); -} - -/** - * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it - * @cs: the cpuset to consider - * @trialcs: trial cpuset - * @buf: buffer of cpu numbers written to this cpuset - */ -static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs, - const char *buf) -{ - int retval; - - /* top_cpuset.cpus_allowed tracks cpu_online_mask; it's read-only */ - if (cs == &top_cpuset) - return -EACCES; - - /* - * An empty cpus_allowed is ok only if the cpuset has no tasks. - * Since cpulist_parse() fails on an empty mask, we special case - * that parsing. The validate_change() call ensures that cpusets - * with tasks have cpus. - */ - if (!*buf) { - cpumask_clear(trialcs->cpus_allowed); - } else { - retval = cpulist_parse(buf, trialcs->cpus_allowed); - if (retval < 0) - return retval; - - if (!cpumask_subset(trialcs->cpus_allowed, - top_cpuset.cpus_allowed)) - return -EINVAL; - } - - /* Nothing to do if the cpus didn't change */ - if (cpumask_equal(cs->cpus_allowed, trialcs->cpus_allowed)) - return 0; - - retval = validate_change(cs, trialcs); - if (retval < 0) - return retval; - - spin_lock_irq(&callback_lock); - cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed); - spin_unlock_irq(&callback_lock); - - /* use trialcs->cpus_allowed as a temp variable */ - update_cpumasks_hier(cs, trialcs->cpus_allowed); - return 0; -} - -/* - * Migrate memory region from one set of nodes to another. This is - * performed asynchronously as it can be called from process migration path - * holding locks involved in process management. All mm migrations are - * performed in the queued order and can be waited for by flushing - * cpuset_migrate_mm_wq. - */ - -struct cpuset_migrate_mm_work { - struct work_struct work; - struct mm_struct *mm; - nodemask_t from; - nodemask_t to; -}; - -static void cpuset_migrate_mm_workfn(struct work_struct *work) -{ - struct cpuset_migrate_mm_work *mwork = - container_of(work, struct cpuset_migrate_mm_work, work); - - /* on a wq worker, no need to worry about %current's mems_allowed */ - do_migrate_pages(mwork->mm, &mwork->from, &mwork->to, MPOL_MF_MOVE_ALL); - mmput(mwork->mm); - kfree(mwork); -} - -static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from, - const nodemask_t *to) -{ - struct cpuset_migrate_mm_work *mwork; - - mwork = kzalloc(sizeof(*mwork), GFP_KERNEL); - if (mwork) { - mwork->mm = mm; - mwork->from = *from; - mwork->to = *to; - INIT_WORK(&mwork->work, cpuset_migrate_mm_workfn); - queue_work(cpuset_migrate_mm_wq, &mwork->work); - } else { - mmput(mm); - } -} - -static void cpuset_post_attach(void) -{ - flush_workqueue(cpuset_migrate_mm_wq); -} - -/* - * cpuset_change_task_nodemask - change task's mems_allowed and mempolicy - * @tsk: the task to change - * @newmems: new nodes that the task will be set - * - * In order to avoid seeing no nodes if the old and new nodes are disjoint, - * we structure updates as setting all new allowed nodes, then clearing newly - * disallowed ones. - */ -static void cpuset_change_task_nodemask(struct task_struct *tsk, - nodemask_t *newmems) -{ - bool need_loop; - - task_lock(tsk); - /* - * Determine if a loop is necessary if another thread is doing - * read_mems_allowed_begin(). If at least one node remains unchanged and - * tsk does not have a mempolicy, then an empty nodemask will not be - * possible when mems_allowed is larger than a word. - */ - need_loop = task_has_mempolicy(tsk) || - !nodes_intersects(*newmems, tsk->mems_allowed); - - if (need_loop) { - local_irq_disable(); - write_seqcount_begin(&tsk->mems_allowed_seq); - } - - nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems); - mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP1); - - mpol_rebind_task(tsk, newmems, MPOL_REBIND_STEP2); - tsk->mems_allowed = *newmems; - - if (need_loop) { - write_seqcount_end(&tsk->mems_allowed_seq); - local_irq_enable(); - } - - task_unlock(tsk); -} - -static void *cpuset_being_rebound; - -/** - * update_tasks_nodemask - Update the nodemasks of tasks in the cpuset. - * @cs: the cpuset in which each task's mems_allowed mask needs to be changed - * - * Iterate through each task of @cs updating its mems_allowed to the - * effective cpuset's. As this function is called with cpuset_mutex held, - * cpuset membership stays stable. - */ -static void update_tasks_nodemask(struct cpuset *cs) -{ - static nodemask_t newmems; /* protected by cpuset_mutex */ - struct css_task_iter it; - struct task_struct *task; - - cpuset_being_rebound = cs; /* causes mpol_dup() rebind */ - - guarantee_online_mems(cs, &newmems); - - /* - * The mpol_rebind_mm() call takes mmap_sem, which we couldn't - * take while holding tasklist_lock. Forks can happen - the - * mpol_dup() cpuset_being_rebound check will catch such forks, - * and rebind their vma mempolicies too. Because we still hold - * the global cpuset_mutex, we know that no other rebind effort - * will be contending for the global variable cpuset_being_rebound. - * It's ok if we rebind the same mm twice; mpol_rebind_mm() - * is idempotent. Also migrate pages in each mm to new nodes. - */ - css_task_iter_start(&cs->css, &it); - while ((task = css_task_iter_next(&it))) { - struct mm_struct *mm; - bool migrate; - - cpuset_change_task_nodemask(task, &newmems); - - mm = get_task_mm(task); - if (!mm) - continue; - - migrate = is_memory_migrate(cs); - - mpol_rebind_mm(mm, &cs->mems_allowed); - if (migrate) - cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems); - else - mmput(mm); - } - css_task_iter_end(&it); - - /* - * All the tasks' nodemasks have been updated, update - * cs->old_mems_allowed. - */ - cs->old_mems_allowed = newmems; - - /* We're done rebinding vmas to this cpuset's new mems_allowed. */ - cpuset_being_rebound = NULL; -} - -/* - * update_nodemasks_hier - Update effective nodemasks and tasks in the subtree - * @cs: the cpuset to consider - * @new_mems: a temp variable for calculating new effective_mems - * - * When configured nodemask is changed, the effective nodemasks of this cpuset - * and all its descendants need to be updated. - * - * On legacy hiearchy, effective_mems will be the same with mems_allowed. - * - * Called with cpuset_mutex held - */ -static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems) -{ - struct cpuset *cp; - struct cgroup_subsys_state *pos_css; - - rcu_read_lock(); - cpuset_for_each_descendant_pre(cp, pos_css, cs) { - struct cpuset *parent = parent_cs(cp); - - nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems); - - /* - * If it becomes empty, inherit the effective mask of the - * parent, which is guaranteed to have some MEMs. - */ - if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && - nodes_empty(*new_mems)) - *new_mems = parent->effective_mems; - - /* Skip the whole subtree if the nodemask remains the same. */ - if (nodes_equal(*new_mems, cp->effective_mems)) { - pos_css = css_rightmost_descendant(pos_css); - continue; - } - - if (!css_tryget_online(&cp->css)) - continue; - rcu_read_unlock(); - - spin_lock_irq(&callback_lock); - cp->effective_mems = *new_mems; - spin_unlock_irq(&callback_lock); - - WARN_ON(!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && - !nodes_equal(cp->mems_allowed, cp->effective_mems)); - - update_tasks_nodemask(cp); - - rcu_read_lock(); - css_put(&cp->css); - } - rcu_read_unlock(); -} - -/* - * Handle user request to change the 'mems' memory placement - * of a cpuset. Needs to validate the request, update the - * cpusets mems_allowed, and for each task in the cpuset, - * update mems_allowed and rebind task's mempolicy and any vma - * mempolicies and if the cpuset is marked 'memory_migrate', - * migrate the tasks pages to the new memory. - * - * Call with cpuset_mutex held. May take callback_lock during call. - * Will take tasklist_lock, scan tasklist for tasks in cpuset cs, - * lock each such tasks mm->mmap_sem, scan its vma's and rebind - * their mempolicies to the cpusets new mems_allowed. - */ -static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs, - const char *buf) -{ - int retval; - - /* - * top_cpuset.mems_allowed tracks node_stats[N_MEMORY]; - * it's read-only - */ - if (cs == &top_cpuset) { - retval = -EACCES; - goto done; - } - - /* - * An empty mems_allowed is ok iff there are no tasks in the cpuset. - * Since nodelist_parse() fails on an empty mask, we special case - * that parsing. The validate_change() call ensures that cpusets - * with tasks have memory. - */ - if (!*buf) { - nodes_clear(trialcs->mems_allowed); - } else { - retval = nodelist_parse(buf, trialcs->mems_allowed); - if (retval < 0) - goto done; - - if (!nodes_subset(trialcs->mems_allowed, - top_cpuset.mems_allowed)) { - retval = -EINVAL; - goto done; - } - } - - if (nodes_equal(cs->mems_allowed, trialcs->mems_allowed)) { - retval = 0; /* Too easy - nothing to do */ - goto done; - } - retval = validate_change(cs, trialcs); - if (retval < 0) - goto done; - - spin_lock_irq(&callback_lock); - cs->mems_allowed = trialcs->mems_allowed; - spin_unlock_irq(&callback_lock); - - /* use trialcs->mems_allowed as a temp variable */ - update_nodemasks_hier(cs, &trialcs->mems_allowed); -done: - return retval; -} - -int current_cpuset_is_being_rebound(void) -{ - int ret; - - rcu_read_lock(); - ret = task_cs(current) == cpuset_being_rebound; - rcu_read_unlock(); - - return ret; -} - -static int update_relax_domain_level(struct cpuset *cs, s64 val) -{ -#ifdef CONFIG_SMP - if (val < -1 || val >= sched_domain_level_max) - return -EINVAL; -#endif - - if (val != cs->relax_domain_level) { - cs->relax_domain_level = val; - if (!cpumask_empty(cs->cpus_allowed) && - is_sched_load_balance(cs)) - rebuild_sched_domains_locked(); - } - - return 0; -} - -/** - * update_tasks_flags - update the spread flags of tasks in the cpuset. - * @cs: the cpuset in which each task's spread flags needs to be changed - * - * Iterate through each task of @cs updating its spread flags. As this - * function is called with cpuset_mutex held, cpuset membership stays - * stable. - */ -static void update_tasks_flags(struct cpuset *cs) -{ - struct css_task_iter it; - struct task_struct *task; - - css_task_iter_start(&cs->css, &it); - while ((task = css_task_iter_next(&it))) - cpuset_update_task_spread_flag(cs, task); - css_task_iter_end(&it); -} - -/* - * update_flag - read a 0 or a 1 in a file and update associated flag - * bit: the bit to update (see cpuset_flagbits_t) - * cs: the cpuset to update - * turning_on: whether the flag is being set or cleared - * - * Call with cpuset_mutex held. - */ - -static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs, - int turning_on) -{ - struct cpuset *trialcs; - int balance_flag_changed; - int spread_flag_changed; - int err; - - trialcs = alloc_trial_cpuset(cs); - if (!trialcs) - return -ENOMEM; - - if (turning_on) - set_bit(bit, &trialcs->flags); - else - clear_bit(bit, &trialcs->flags); - - err = validate_change(cs, trialcs); - if (err < 0) - goto out; - - balance_flag_changed = (is_sched_load_balance(cs) != - is_sched_load_balance(trialcs)); - - spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs)) - || (is_spread_page(cs) != is_spread_page(trialcs))); - - spin_lock_irq(&callback_lock); - cs->flags = trialcs->flags; - spin_unlock_irq(&callback_lock); - - if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed) - rebuild_sched_domains_locked(); - - if (spread_flag_changed) - update_tasks_flags(cs); -out: - free_trial_cpuset(trialcs); - return err; -} - -/* - * Frequency meter - How fast is some event occurring? - * - * These routines manage a digitally filtered, constant time based, - * event frequency meter. There are four routines: - * fmeter_init() - initialize a frequency meter. - * fmeter_markevent() - called each time the event happens. - * fmeter_getrate() - returns the recent rate of such events. - * fmeter_update() - internal routine used to update fmeter. - * - * A common data structure is passed to each of these routines, - * which is used to keep track of the state required to manage the - * frequency meter and its digital filter. - * - * The filter works on the number of events marked per unit time. - * The filter is single-pole low-pass recursive (IIR). The time unit - * is 1 second. Arithmetic is done using 32-bit integers scaled to - * simulate 3 decimal digits of precision (multiplied by 1000). - * - * With an FM_COEF of 933, and a time base of 1 second, the filter - * has a half-life of 10 seconds, meaning that if the events quit - * happening, then the rate returned from the fmeter_getrate() - * will be cut in half each 10 seconds, until it converges to zero. - * - * It is not worth doing a real infinitely recursive filter. If more - * than FM_MAXTICKS ticks have elapsed since the last filter event, - * just compute FM_MAXTICKS ticks worth, by which point the level - * will be stable. - * - * Limit the count of unprocessed events to FM_MAXCNT, so as to avoid - * arithmetic overflow in the fmeter_update() routine. - * - * Given the simple 32 bit integer arithmetic used, this meter works - * best for reporting rates between one per millisecond (msec) and - * one per 32 (approx) seconds. At constant rates faster than one - * per msec it maxes out at values just under 1,000,000. At constant - * rates between one per msec, and one per second it will stabilize - * to a value N*1000, where N is the rate of events per second. - * At constant rates between one per second and one per 32 seconds, - * it will be choppy, moving up on the seconds that have an event, - * and then decaying until the next event. At rates slower than - * about one in 32 seconds, it decays all the way back to zero between - * each event. - */ - -#define FM_COEF 933 /* coefficient for half-life of 10 secs */ -#define FM_MAXTICKS ((u32)99) /* useless computing more ticks than this */ -#define FM_MAXCNT 1000000 /* limit cnt to avoid overflow */ -#define FM_SCALE 1000 /* faux fixed point scale */ - -/* Initialize a frequency meter */ -static void fmeter_init(struct fmeter *fmp) -{ - fmp->cnt = 0; - fmp->val = 0; - fmp->time = 0; - spin_lock_init(&fmp->lock); -} - -/* Internal meter update - process cnt events and update value */ -static void fmeter_update(struct fmeter *fmp) -{ - time64_t now; - u32 ticks; - - now = ktime_get_seconds(); - ticks = now - fmp->time; - - if (ticks == 0) - return; - - ticks = min(FM_MAXTICKS, ticks); - while (ticks-- > 0) - fmp->val = (FM_COEF * fmp->val) / FM_SCALE; - fmp->time = now; - - fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE; - fmp->cnt = 0; -} - -/* Process any previous ticks, then bump cnt by one (times scale). */ -static void fmeter_markevent(struct fmeter *fmp) -{ - spin_lock(&fmp->lock); - fmeter_update(fmp); - fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE); - spin_unlock(&fmp->lock); -} - -/* Process any previous ticks, then return current value. */ -static int fmeter_getrate(struct fmeter *fmp) -{ - int val; - - spin_lock(&fmp->lock); - fmeter_update(fmp); - val = fmp->val; - spin_unlock(&fmp->lock); - return val; -} - -static struct cpuset *cpuset_attach_old_cs; - -/* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */ -static int cpuset_can_attach(struct cgroup_taskset *tset) -{ - struct cgroup_subsys_state *css; - struct cpuset *cs; - struct task_struct *task; - int ret; - - /* used later by cpuset_attach() */ - cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset, &css)); - cs = css_cs(css); - - mutex_lock(&cpuset_mutex); - - /* allow moving tasks into an empty cpuset if on default hierarchy */ - ret = -ENOSPC; - if (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) && - (cpumask_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed))) - goto out_unlock; - - cgroup_taskset_for_each(task, css, tset) { - ret = task_can_attach(task, cs->cpus_allowed); - if (ret) - goto out_unlock; - ret = security_task_setscheduler(task); - if (ret) - goto out_unlock; - } - - /* - * Mark attach is in progress. This makes validate_change() fail - * changes which zero cpus/mems_allowed. - */ - cs->attach_in_progress++; - ret = 0; -out_unlock: - mutex_unlock(&cpuset_mutex); - return ret; -} - -static void cpuset_cancel_attach(struct cgroup_taskset *tset) -{ - struct cgroup_subsys_state *css; - struct cpuset *cs; - - cgroup_taskset_first(tset, &css); - cs = css_cs(css); - - mutex_lock(&cpuset_mutex); - css_cs(css)->attach_in_progress--; - mutex_unlock(&cpuset_mutex); -} - -/* - * Protected by cpuset_mutex. cpus_attach is used only by cpuset_attach() - * but we can't allocate it dynamically there. Define it global and - * allocate from cpuset_init(). - */ -static cpumask_var_t cpus_attach; - -static void cpuset_attach(struct cgroup_taskset *tset) -{ - /* static buf protected by cpuset_mutex */ - static nodemask_t cpuset_attach_nodemask_to; - struct task_struct *task; - struct task_struct *leader; - struct cgroup_subsys_state *css; - struct cpuset *cs; - struct cpuset *oldcs = cpuset_attach_old_cs; - - cgroup_taskset_first(tset, &css); - cs = css_cs(css); - - mutex_lock(&cpuset_mutex); - - /* prepare for attach */ - if (cs == &top_cpuset) - cpumask_copy(cpus_attach, cpu_possible_mask); - else - guarantee_online_cpus(cs, cpus_attach); - - guarantee_online_mems(cs, &cpuset_attach_nodemask_to); - - cgroup_taskset_for_each(task, css, tset) { - /* - * can_attach beforehand should guarantee that this doesn't - * fail. TODO: have a better way to handle failure here - */ - WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach)); - - cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to); - cpuset_update_task_spread_flag(cs, task); - } - - /* - * Change mm for all threadgroup leaders. This is expensive and may - * sleep and should be moved outside migration path proper. - */ - cpuset_attach_nodemask_to = cs->effective_mems; - cgroup_taskset_for_each_leader(leader, css, tset) { - struct mm_struct *mm = get_task_mm(leader); - - if (mm) { - mpol_rebind_mm(mm, &cpuset_attach_nodemask_to); - - /* - * old_mems_allowed is the same with mems_allowed - * here, except if this task is being moved - * automatically due to hotplug. In that case - * @mems_allowed has been updated and is empty, so - * @old_mems_allowed is the right nodesets that we - * migrate mm from. - */ - if (is_memory_migrate(cs)) - cpuset_migrate_mm(mm, &oldcs->old_mems_allowed, - &cpuset_attach_nodemask_to); - else - mmput(mm); - } - } - - cs->old_mems_allowed = cpuset_attach_nodemask_to; - - cs->attach_in_progress--; - if (!cs->attach_in_progress) - wake_up(&cpuset_attach_wq); - - mutex_unlock(&cpuset_mutex); -} - -/* The various types of files and directories in a cpuset file system */ - -typedef enum { - FILE_MEMORY_MIGRATE, - FILE_CPULIST, - FILE_MEMLIST, - FILE_EFFECTIVE_CPULIST, - FILE_EFFECTIVE_MEMLIST, - FILE_CPU_EXCLUSIVE, - FILE_MEM_EXCLUSIVE, - FILE_MEM_HARDWALL, - FILE_SCHED_LOAD_BALANCE, - FILE_SCHED_RELAX_DOMAIN_LEVEL, - FILE_MEMORY_PRESSURE_ENABLED, - FILE_MEMORY_PRESSURE, - FILE_SPREAD_PAGE, - FILE_SPREAD_SLAB, -} cpuset_filetype_t; - -static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft, - u64 val) -{ - struct cpuset *cs = css_cs(css); - cpuset_filetype_t type = cft->private; - int retval = 0; - - mutex_lock(&cpuset_mutex); - if (!is_cpuset_online(cs)) { - retval = -ENODEV; - goto out_unlock; - } - - switch (type) { - case FILE_CPU_EXCLUSIVE: - retval = update_flag(CS_CPU_EXCLUSIVE, cs, val); - break; - case FILE_MEM_EXCLUSIVE: - retval = update_flag(CS_MEM_EXCLUSIVE, cs, val); - break; - case FILE_MEM_HARDWALL: - retval = update_flag(CS_MEM_HARDWALL, cs, val); - break; - case FILE_SCHED_LOAD_BALANCE: - retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, val); - break; - case FILE_MEMORY_MIGRATE: - retval = update_flag(CS_MEMORY_MIGRATE, cs, val); - break; - case FILE_MEMORY_PRESSURE_ENABLED: - cpuset_memory_pressure_enabled = !!val; - break; - case FILE_SPREAD_PAGE: - retval = update_flag(CS_SPREAD_PAGE, cs, val); - break; - case FILE_SPREAD_SLAB: - retval = update_flag(CS_SPREAD_SLAB, cs, val); - break; - default: - retval = -EINVAL; - break; - } -out_unlock: - mutex_unlock(&cpuset_mutex); - return retval; -} - -static int cpuset_write_s64(struct cgroup_subsys_state *css, struct cftype *cft, - s64 val) -{ - struct cpuset *cs = css_cs(css); - cpuset_filetype_t type = cft->private; - int retval = -ENODEV; - - mutex_lock(&cpuset_mutex); - if (!is_cpuset_online(cs)) - goto out_unlock; - - switch (type) { - case FILE_SCHED_RELAX_DOMAIN_LEVEL: - retval = update_relax_domain_level(cs, val); - break; - default: - retval = -EINVAL; - break; - } -out_unlock: - mutex_unlock(&cpuset_mutex); - return retval; -} - -/* - * Common handling for a write to a "cpus" or "mems" file. - */ -static ssize_t cpuset_write_resmask(struct kernfs_open_file *of, - char *buf, size_t nbytes, loff_t off) -{ - struct cpuset *cs = css_cs(of_css(of)); - struct cpuset *trialcs; - int retval = -ENODEV; - - buf = strstrip(buf); - - /* - * CPU or memory hotunplug may leave @cs w/o any execution - * resources, in which case the hotplug code asynchronously updates - * configuration and transfers all tasks to the nearest ancestor - * which can execute. - * - * As writes to "cpus" or "mems" may restore @cs's execution - * resources, wait for the previously scheduled operations before - * proceeding, so that we don't end up keep removing tasks added - * after execution capability is restored. - * - * cpuset_hotplug_work calls back into cgroup core via - * cgroup_transfer_tasks() and waiting for it from a cgroupfs - * operation like this one can lead to a deadlock through kernfs - * active_ref protection. Let's break the protection. Losing the - * protection is okay as we check whether @cs is online after - * grabbing cpuset_mutex anyway. This only happens on the legacy - * hierarchies. - */ - css_get(&cs->css); - kernfs_break_active_protection(of->kn); - flush_work(&cpuset_hotplug_work); - - mutex_lock(&cpuset_mutex); - if (!is_cpuset_online(cs)) - goto out_unlock; - - trialcs = alloc_trial_cpuset(cs); - if (!trialcs) { - retval = -ENOMEM; - goto out_unlock; - } - - switch (of_cft(of)->private) { - case FILE_CPULIST: - retval = update_cpumask(cs, trialcs, buf); - break; - case FILE_MEMLIST: - retval = update_nodemask(cs, trialcs, buf); - break; - default: - retval = -EINVAL; - break; - } - - free_trial_cpuset(trialcs); -out_unlock: - mutex_unlock(&cpuset_mutex); - kernfs_unbreak_active_protection(of->kn); - css_put(&cs->css); - flush_workqueue(cpuset_migrate_mm_wq); - return retval ?: nbytes; -} - -/* - * These ascii lists should be read in a single call, by using a user - * buffer large enough to hold the entire map. If read in smaller - * chunks, there is no guarantee of atomicity. Since the display format - * used, list of ranges of sequential numbers, is variable length, - * and since these maps can change value dynamically, one could read - * gibberish by doing partial reads while a list was changing. - */ -static int cpuset_common_seq_show(struct seq_file *sf, void *v) -{ - struct cpuset *cs = css_cs(seq_css(sf)); - cpuset_filetype_t type = seq_cft(sf)->private; - int ret = 0; - - spin_lock_irq(&callback_lock); - - switch (type) { - case FILE_CPULIST: - seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->cpus_allowed)); - break; - case FILE_MEMLIST: - seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->mems_allowed)); - break; - case FILE_EFFECTIVE_CPULIST: - seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_cpus)); - break; - case FILE_EFFECTIVE_MEMLIST: - seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->effective_mems)); - break; - default: - ret = -EINVAL; - } - - spin_unlock_irq(&callback_lock); - return ret; -} - -static u64 cpuset_read_u64(struct cgroup_subsys_state *css, struct cftype *cft) -{ - struct cpuset *cs = css_cs(css); - cpuset_filetype_t type = cft->private; - switch (type) { - case FILE_CPU_EXCLUSIVE: - return is_cpu_exclusive(cs); - case FILE_MEM_EXCLUSIVE: - return is_mem_exclusive(cs); - case FILE_MEM_HARDWALL: - return is_mem_hardwall(cs); - case FILE_SCHED_LOAD_BALANCE: - return is_sched_load_balance(cs); - case FILE_MEMORY_MIGRATE: - return is_memory_migrate(cs); - case FILE_MEMORY_PRESSURE_ENABLED: - return cpuset_memory_pressure_enabled; - case FILE_MEMORY_PRESSURE: - return fmeter_getrate(&cs->fmeter); - case FILE_SPREAD_PAGE: - return is_spread_page(cs); - case FILE_SPREAD_SLAB: - return is_spread_slab(cs); - default: - BUG(); - } - - /* Unreachable but makes gcc happy */ - return 0; -} - -static s64 cpuset_read_s64(struct cgroup_subsys_state *css, struct cftype *cft) -{ - struct cpuset *cs = css_cs(css); - cpuset_filetype_t type = cft->private; - switch (type) { - case FILE_SCHED_RELAX_DOMAIN_LEVEL: - return cs->relax_domain_level; - default: - BUG(); - } - - /* Unrechable but makes gcc happy */ - return 0; -} - - -/* - * for the common functions, 'private' gives the type of file - */ - -static struct cftype files[] = { - { - .name = "cpus", - .seq_show = cpuset_common_seq_show, - .write = cpuset_write_resmask, - .max_write_len = (100U + 6 * NR_CPUS), - .private = FILE_CPULIST, - }, - - { - .name = "mems", - .seq_show = cpuset_common_seq_show, - .write = cpuset_write_resmask, - .max_write_len = (100U + 6 * MAX_NUMNODES), - .private = FILE_MEMLIST, - }, - - { - .name = "effective_cpus", - .seq_show = cpuset_common_seq_show, - .private = FILE_EFFECTIVE_CPULIST, - }, - - { - .name = "effective_mems", - .seq_show = cpuset_common_seq_show, - .private = FILE_EFFECTIVE_MEMLIST, - }, - - { - .name = "cpu_exclusive", - .read_u64 = cpuset_read_u64, - .write_u64 = cpuset_write_u64, - .private = FILE_CPU_EXCLUSIVE, - }, - - { - .name = "mem_exclusive", - .read_u64 = cpuset_read_u64, - .write_u64 = cpuset_write_u64, - .private = FILE_MEM_EXCLUSIVE, - }, - - { - .name = "mem_hardwall", - .read_u64 = cpuset_read_u64, - .write_u64 = cpuset_write_u64, - .private = FILE_MEM_HARDWALL, - }, - - { - .name = "sched_load_balance", - .read_u64 = cpuset_read_u64, - .write_u64 = cpuset_write_u64, - .private = FILE_SCHED_LOAD_BALANCE, - }, - - { - .name = "sched_relax_domain_level", - .read_s64 = cpuset_read_s64, - .write_s64 = cpuset_write_s64, - .private = FILE_SCHED_RELAX_DOMAIN_LEVEL, - }, - - { - .name = "memory_migrate", - .read_u64 = cpuset_read_u64, - .write_u64 = cpuset_write_u64, - .private = FILE_MEMORY_MIGRATE, - }, - - { - .name = "memory_pressure", - .read_u64 = cpuset_read_u64, - }, - - { - .name = "memory_spread_page", - .read_u64 = cpuset_read_u64, - .write_u64 = cpuset_write_u64, - .private = FILE_SPREAD_PAGE, - }, - - { - .name = "memory_spread_slab", - .read_u64 = cpuset_read_u64, - .write_u64 = cpuset_write_u64, - .private = FILE_SPREAD_SLAB, - }, - - { - .name = "memory_pressure_enabled", - .flags = CFTYPE_ONLY_ON_ROOT, - .read_u64 = cpuset_read_u64, - .write_u64 = cpuset_write_u64, - .private = FILE_MEMORY_PRESSURE_ENABLED, - }, - - { } /* terminate */ -}; - -/* - * cpuset_css_alloc - allocate a cpuset css - * cgrp: control group that the new cpuset will be part of - */ - -static struct cgroup_subsys_state * -cpuset_css_alloc(struct cgroup_subsys_state *parent_css) -{ - struct cpuset *cs; - - if (!parent_css) - return &top_cpuset.css; - - cs = kzalloc(sizeof(*cs), GFP_KERNEL); - if (!cs) - return ERR_PTR(-ENOMEM); - if (!alloc_cpumask_var(&cs->cpus_allowed, GFP_KERNEL)) - goto free_cs; - if (!alloc_cpumask_var(&cs->effective_cpus, GFP_KERNEL)) - goto free_cpus; - - set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); - cpumask_clear(cs->cpus_allowed); - nodes_clear(cs->mems_allowed); - cpumask_clear(cs->effective_cpus); - nodes_clear(cs->effective_mems); - fmeter_init(&cs->fmeter); - cs->relax_domain_level = -1; - - return &cs->css; - -free_cpus: - free_cpumask_var(cs->cpus_allowed); -free_cs: - kfree(cs); - return ERR_PTR(-ENOMEM); -} - -static int cpuset_css_online(struct cgroup_subsys_state *css) -{ - struct cpuset *cs = css_cs(css); - struct cpuset *parent = parent_cs(cs); - struct cpuset *tmp_cs; - struct cgroup_subsys_state *pos_css; - - if (!parent) - return 0; - - mutex_lock(&cpuset_mutex); - - set_bit(CS_ONLINE, &cs->flags); - if (is_spread_page(parent)) - set_bit(CS_SPREAD_PAGE, &cs->flags); - if (is_spread_slab(parent)) - set_bit(CS_SPREAD_SLAB, &cs->flags); - - cpuset_inc(); - - spin_lock_irq(&callback_lock); - if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys)) { - cpumask_copy(cs->effective_cpus, parent->effective_cpus); - cs->effective_mems = parent->effective_mems; - } - spin_unlock_irq(&callback_lock); - - if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags)) - goto out_unlock; - - /* - * Clone @parent's configuration if CGRP_CPUSET_CLONE_CHILDREN is - * set. This flag handling is implemented in cgroup core for - * histrical reasons - the flag may be specified during mount. - * - * Currently, if any sibling cpusets have exclusive cpus or mem, we - * refuse to clone the configuration - thereby refusing the task to - * be entered, and as a result refusing the sys_unshare() or - * clone() which initiated it. If this becomes a problem for some - * users who wish to allow that scenario, then this could be - * changed to grant parent->cpus_allowed-sibling_cpus_exclusive - * (and likewise for mems) to the new cgroup. - */ - rcu_read_lock(); - cpuset_for_each_child(tmp_cs, pos_css, parent) { - if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs)) { - rcu_read_unlock(); - goto out_unlock; - } - } - rcu_read_unlock(); - - spin_lock_irq(&callback_lock); - cs->mems_allowed = parent->mems_allowed; - cs->effective_mems = parent->mems_allowed; - cpumask_copy(cs->cpus_allowed, parent->cpus_allowed); - cpumask_copy(cs->effective_cpus, parent->cpus_allowed); - spin_unlock_irq(&callback_lock); -out_unlock: - mutex_unlock(&cpuset_mutex); - return 0; -} - -/* - * If the cpuset being removed has its flag 'sched_load_balance' - * enabled, then simulate turning sched_load_balance off, which - * will call rebuild_sched_domains_locked(). - */ - -static void cpuset_css_offline(struct cgroup_subsys_state *css) -{ - struct cpuset *cs = css_cs(css); - - mutex_lock(&cpuset_mutex); - - if (is_sched_load_balance(cs)) - update_flag(CS_SCHED_LOAD_BALANCE, cs, 0); - - cpuset_dec(); - clear_bit(CS_ONLINE, &cs->flags); - - mutex_unlock(&cpuset_mutex); -} - -static void cpuset_css_free(struct cgroup_subsys_state *css) -{ - struct cpuset *cs = css_cs(css); - - free_cpumask_var(cs->effective_cpus); - free_cpumask_var(cs->cpus_allowed); - kfree(cs); -} - -static void cpuset_bind(struct cgroup_subsys_state *root_css) -{ - mutex_lock(&cpuset_mutex); - spin_lock_irq(&callback_lock); - - if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys)) { - cpumask_copy(top_cpuset.cpus_allowed, cpu_possible_mask); - top_cpuset.mems_allowed = node_possible_map; - } else { - cpumask_copy(top_cpuset.cpus_allowed, - top_cpuset.effective_cpus); - top_cpuset.mems_allowed = top_cpuset.effective_mems; - } - - spin_unlock_irq(&callback_lock); - mutex_unlock(&cpuset_mutex); -} - -/* - * Make sure the new task conform to the current state of its parent, - * which could have been changed by cpuset just after it inherits the - * state from the parent and before it sits on the cgroup's task list. - */ -static void cpuset_fork(struct task_struct *task) -{ - if (task_css_is_root(task, cpuset_cgrp_id)) - return; - - set_cpus_allowed_ptr(task, ¤t->cpus_allowed); - task->mems_allowed = current->mems_allowed; -} - -struct cgroup_subsys cpuset_cgrp_subsys = { - .css_alloc = cpuset_css_alloc, - .css_online = cpuset_css_online, - .css_offline = cpuset_css_offline, - .css_free = cpuset_css_free, - .can_attach = cpuset_can_attach, - .cancel_attach = cpuset_cancel_attach, - .attach = cpuset_attach, - .post_attach = cpuset_post_attach, - .bind = cpuset_bind, - .fork = cpuset_fork, - .legacy_cftypes = files, - .early_init = true, -}; - -/** - * cpuset_init - initialize cpusets at system boot - * - * Description: Initialize top_cpuset and the cpuset internal file system, - **/ - -int __init cpuset_init(void) -{ - int err = 0; - - if (!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL)) - BUG(); - if (!alloc_cpumask_var(&top_cpuset.effective_cpus, GFP_KERNEL)) - BUG(); - - cpumask_setall(top_cpuset.cpus_allowed); - nodes_setall(top_cpuset.mems_allowed); - cpumask_setall(top_cpuset.effective_cpus); - nodes_setall(top_cpuset.effective_mems); - - fmeter_init(&top_cpuset.fmeter); - set_bit(CS_SCHED_LOAD_BALANCE, &top_cpuset.flags); - top_cpuset.relax_domain_level = -1; - - err = register_filesystem(&cpuset_fs_type); - if (err < 0) - return err; - - if (!alloc_cpumask_var(&cpus_attach, GFP_KERNEL)) - BUG(); - - return 0; -} - -/* - * If CPU and/or memory hotplug handlers, below, unplug any CPUs - * or memory nodes, we need to walk over the cpuset hierarchy, - * removing that CPU or node from all cpusets. If this removes the - * last CPU or node from a cpuset, then move the tasks in the empty - * cpuset to its next-highest non-empty parent. - */ -static void remove_tasks_in_empty_cpuset(struct cpuset *cs) -{ - struct cpuset *parent; - - /* - * Find its next-highest non-empty parent, (top cpuset - * has online cpus, so can't be empty). - */ - parent = parent_cs(cs); - while (cpumask_empty(parent->cpus_allowed) || - nodes_empty(parent->mems_allowed)) - parent = parent_cs(parent); - - if (cgroup_transfer_tasks(parent->css.cgroup, cs->css.cgroup)) { - pr_err("cpuset: failed to transfer tasks out of empty cpuset "); - pr_cont_cgroup_name(cs->css.cgroup); - pr_cont("\n"); - } -} - -static void -hotplug_update_tasks_legacy(struct cpuset *cs, - struct cpumask *new_cpus, nodemask_t *new_mems, - bool cpus_updated, bool mems_updated) -{ - bool is_empty; - - spin_lock_irq(&callback_lock); - cpumask_copy(cs->cpus_allowed, new_cpus); - cpumask_copy(cs->effective_cpus, new_cpus); - cs->mems_allowed = *new_mems; - cs->effective_mems = *new_mems; - spin_unlock_irq(&callback_lock); - - /* - * Don't call update_tasks_cpumask() if the cpuset becomes empty, - * as the tasks will be migratecd to an ancestor. - */ - if (cpus_updated && !cpumask_empty(cs->cpus_allowed)) - update_tasks_cpumask(cs); - if (mems_updated && !nodes_empty(cs->mems_allowed)) - update_tasks_nodemask(cs); - - is_empty = cpumask_empty(cs->cpus_allowed) || - nodes_empty(cs->mems_allowed); - - mutex_unlock(&cpuset_mutex); - - /* - * Move tasks to the nearest ancestor with execution resources, - * This is full cgroup operation which will also call back into - * cpuset. Should be done outside any lock. - */ - if (is_empty) - remove_tasks_in_empty_cpuset(cs); - - mutex_lock(&cpuset_mutex); -} - -static void -hotplug_update_tasks(struct cpuset *cs, - struct cpumask *new_cpus, nodemask_t *new_mems, - bool cpus_updated, bool mems_updated) -{ - if (cpumask_empty(new_cpus)) - cpumask_copy(new_cpus, parent_cs(cs)->effective_cpus); - if (nodes_empty(*new_mems)) - *new_mems = parent_cs(cs)->effective_mems; - - spin_lock_irq(&callback_lock); - cpumask_copy(cs->effective_cpus, new_cpus); - cs->effective_mems = *new_mems; - spin_unlock_irq(&callback_lock); - - if (cpus_updated) - update_tasks_cpumask(cs); - if (mems_updated) - update_tasks_nodemask(cs); -} - -/** - * cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug - * @cs: cpuset in interest - * - * Compare @cs's cpu and mem masks against top_cpuset and if some have gone - * offline, update @cs accordingly. If @cs ends up with no CPU or memory, - * all its tasks are moved to the nearest ancestor with both resources. - */ -static void cpuset_hotplug_update_tasks(struct cpuset *cs) -{ - static cpumask_t new_cpus; - static nodemask_t new_mems; - bool cpus_updated; - bool mems_updated; -retry: - wait_event(cpuset_attach_wq, cs->attach_in_progress == 0); - - mutex_lock(&cpuset_mutex); - - /* - * We have raced with task attaching. We wait until attaching - * is finished, so we won't attach a task to an empty cpuset. - */ - if (cs->attach_in_progress) { - mutex_unlock(&cpuset_mutex); - goto retry; - } - - cpumask_and(&new_cpus, cs->cpus_allowed, parent_cs(cs)->effective_cpus); - nodes_and(new_mems, cs->mems_allowed, parent_cs(cs)->effective_mems); - - cpus_updated = !cpumask_equal(&new_cpus, cs->effective_cpus); - mems_updated = !nodes_equal(new_mems, cs->effective_mems); - - if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys)) - hotplug_update_tasks(cs, &new_cpus, &new_mems, - cpus_updated, mems_updated); - else - hotplug_update_tasks_legacy(cs, &new_cpus, &new_mems, - cpus_updated, mems_updated); - - mutex_unlock(&cpuset_mutex); -} - -/** - * cpuset_hotplug_workfn - handle CPU/memory hotunplug for a cpuset - * - * This function is called after either CPU or memory configuration has - * changed and updates cpuset accordingly. The top_cpuset is always - * synchronized to cpu_active_mask and N_MEMORY, which is necessary in - * order to make cpusets transparent (of no affect) on systems that are - * actively using CPU hotplug but making no active use of cpusets. - * - * Non-root cpusets are only affected by offlining. If any CPUs or memory - * nodes have been taken down, cpuset_hotplug_update_tasks() is invoked on - * all descendants. - * - * Note that CPU offlining during suspend is ignored. We don't modify - * cpusets across suspend/resume cycles at all. - */ -static void cpuset_hotplug_workfn(struct work_struct *work) -{ - static cpumask_t new_cpus; - static nodemask_t new_mems; - bool cpus_updated, mems_updated; - bool on_dfl = cgroup_subsys_on_dfl(cpuset_cgrp_subsys); - - mutex_lock(&cpuset_mutex); - - /* fetch the available cpus/mems and find out which changed how */ - cpumask_copy(&new_cpus, cpu_active_mask); - new_mems = node_states[N_MEMORY]; - - cpus_updated = !cpumask_equal(top_cpuset.effective_cpus, &new_cpus); - mems_updated = !nodes_equal(top_cpuset.effective_mems, new_mems); - - /* synchronize cpus_allowed to cpu_active_mask */ - if (cpus_updated) { - spin_lock_irq(&callback_lock); - if (!on_dfl) - cpumask_copy(top_cpuset.cpus_allowed, &new_cpus); - cpumask_copy(top_cpuset.effective_cpus, &new_cpus); - spin_unlock_irq(&callback_lock); - /* we don't mess with cpumasks of tasks in top_cpuset */ - } - - /* synchronize mems_allowed to N_MEMORY */ - if (mems_updated) { - spin_lock_irq(&callback_lock); - if (!on_dfl) - top_cpuset.mems_allowed = new_mems; - top_cpuset.effective_mems = new_mems; - spin_unlock_irq(&callback_lock); - update_tasks_nodemask(&top_cpuset); - } - - mutex_unlock(&cpuset_mutex); - - /* if cpus or mems changed, we need to propagate to descendants */ - if (cpus_updated || mems_updated) { - struct cpuset *cs; - struct cgroup_subsys_state *pos_css; - - rcu_read_lock(); - cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) { - if (cs == &top_cpuset || !css_tryget_online(&cs->css)) - continue; - rcu_read_unlock(); - - cpuset_hotplug_update_tasks(cs); - - rcu_read_lock(); - css_put(&cs->css); - } - rcu_read_unlock(); - } - - /* rebuild sched domains if cpus_allowed has changed */ - if (cpus_updated) - rebuild_sched_domains(); -} - -void cpuset_update_active_cpus(bool cpu_online) -{ - /* - * We're inside cpu hotplug critical region which usually nests - * inside cgroup synchronization. Bounce actual hotplug processing - * to a work item to avoid reverse locking order. - * - * We still need to do partition_sched_domains() synchronously; - * otherwise, the scheduler will get confused and put tasks to the - * dead CPU. Fall back to the default single domain. - * cpuset_hotplug_workfn() will rebuild it as necessary. - */ - partition_sched_domains(1, NULL, NULL); - schedule_work(&cpuset_hotplug_work); -} - -/* - * Keep top_cpuset.mems_allowed tracking node_states[N_MEMORY]. - * Call this routine anytime after node_states[N_MEMORY] changes. - * See cpuset_update_active_cpus() for CPU hotplug handling. - */ -static int cpuset_track_online_nodes(struct notifier_block *self, - unsigned long action, void *arg) -{ - schedule_work(&cpuset_hotplug_work); - return NOTIFY_OK; -} - -static struct notifier_block cpuset_track_online_nodes_nb = { - .notifier_call = cpuset_track_online_nodes, - .priority = 10, /* ??! */ -}; - -/** - * cpuset_init_smp - initialize cpus_allowed - * - * Description: Finish top cpuset after cpu, node maps are initialized - */ -void __init cpuset_init_smp(void) -{ - cpumask_copy(top_cpuset.cpus_allowed, cpu_active_mask); - top_cpuset.mems_allowed = node_states[N_MEMORY]; - top_cpuset.old_mems_allowed = top_cpuset.mems_allowed; - - cpumask_copy(top_cpuset.effective_cpus, cpu_active_mask); - top_cpuset.effective_mems = node_states[N_MEMORY]; - - register_hotmemory_notifier(&cpuset_track_online_nodes_nb); - - cpuset_migrate_mm_wq = alloc_ordered_workqueue("cpuset_migrate_mm", 0); - BUG_ON(!cpuset_migrate_mm_wq); -} - -/** - * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset. - * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed. - * @pmask: pointer to struct cpumask variable to receive cpus_allowed set. - * - * Description: Returns the cpumask_var_t cpus_allowed of the cpuset - * attached to the specified @tsk. Guaranteed to return some non-empty - * subset of cpu_online_mask, even if this means going outside the - * tasks cpuset. - **/ - -void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask) -{ - unsigned long flags; - - spin_lock_irqsave(&callback_lock, flags); - rcu_read_lock(); - guarantee_online_cpus(task_cs(tsk), pmask); - rcu_read_unlock(); - spin_unlock_irqrestore(&callback_lock, flags); -} - -void cpuset_cpus_allowed_fallback(struct task_struct *tsk) -{ - rcu_read_lock(); - do_set_cpus_allowed(tsk, task_cs(tsk)->effective_cpus); - rcu_read_unlock(); - - /* - * We own tsk->cpus_allowed, nobody can change it under us. - * - * But we used cs && cs->cpus_allowed lockless and thus can - * race with cgroup_attach_task() or update_cpumask() and get - * the wrong tsk->cpus_allowed. However, both cases imply the - * subsequent cpuset_change_cpumask()->set_cpus_allowed_ptr() - * which takes task_rq_lock(). - * - * If we are called after it dropped the lock we must see all - * changes in tsk_cs()->cpus_allowed. Otherwise we can temporary - * set any mask even if it is not right from task_cs() pov, - * the pending set_cpus_allowed_ptr() will fix things. - * - * select_fallback_rq() will fix things ups and set cpu_possible_mask - * if required. - */ -} - -void __init cpuset_init_current_mems_allowed(void) -{ - nodes_setall(current->mems_allowed); -} - -/** - * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset. - * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed. - * - * Description: Returns the nodemask_t mems_allowed of the cpuset - * attached to the specified @tsk. Guaranteed to return some non-empty - * subset of node_states[N_MEMORY], even if this means going outside the - * tasks cpuset. - **/ - -nodemask_t cpuset_mems_allowed(struct task_struct *tsk) -{ - nodemask_t mask; - unsigned long flags; - - spin_lock_irqsave(&callback_lock, flags); - rcu_read_lock(); - guarantee_online_mems(task_cs(tsk), &mask); - rcu_read_unlock(); - spin_unlock_irqrestore(&callback_lock, flags); - - return mask; -} - -/** - * cpuset_nodemask_valid_mems_allowed - check nodemask vs. curremt mems_allowed - * @nodemask: the nodemask to be checked - * - * Are any of the nodes in the nodemask allowed in current->mems_allowed? - */ -int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) -{ - return nodes_intersects(*nodemask, current->mems_allowed); -} - -/* - * nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or - * mem_hardwall ancestor to the specified cpuset. Call holding - * callback_lock. If no ancestor is mem_exclusive or mem_hardwall - * (an unusual configuration), then returns the root cpuset. - */ -static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs) -{ - while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs)) - cs = parent_cs(cs); - return cs; -} - -/** - * cpuset_node_allowed - Can we allocate on a memory node? - * @node: is this an allowed node? - * @gfp_mask: memory allocation flags - * - * If we're in interrupt, yes, we can always allocate. If @node is set in - * current's mems_allowed, yes. If it's not a __GFP_HARDWALL request and this - * node is set in the nearest hardwalled cpuset ancestor to current's cpuset, - * yes. If current has access to memory reserves due to TIF_MEMDIE, yes. - * Otherwise, no. - * - * GFP_USER allocations are marked with the __GFP_HARDWALL bit, - * and do not allow allocations outside the current tasks cpuset - * unless the task has been OOM killed as is marked TIF_MEMDIE. - * GFP_KERNEL allocations are not so marked, so can escape to the - * nearest enclosing hardwalled ancestor cpuset. - * - * Scanning up parent cpusets requires callback_lock. The - * __alloc_pages() routine only calls here with __GFP_HARDWALL bit - * _not_ set if it's a GFP_KERNEL allocation, and all nodes in the - * current tasks mems_allowed came up empty on the first pass over - * the zonelist. So only GFP_KERNEL allocations, if all nodes in the - * cpuset are short of memory, might require taking the callback_lock. - * - * The first call here from mm/page_alloc:get_page_from_freelist() - * has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets, - * so no allocation on a node outside the cpuset is allowed (unless - * in interrupt, of course). - * - * The second pass through get_page_from_freelist() doesn't even call - * here for GFP_ATOMIC calls. For those calls, the __alloc_pages() - * variable 'wait' is not set, and the bit ALLOC_CPUSET is not set - * in alloc_flags. That logic and the checks below have the combined - * affect that: - * in_interrupt - any node ok (current task context irrelevant) - * GFP_ATOMIC - any node ok - * TIF_MEMDIE - any node ok - * GFP_KERNEL - any node in enclosing hardwalled cpuset ok - * GFP_USER - only nodes in current tasks mems allowed ok. - */ -bool __cpuset_node_allowed(int node, gfp_t gfp_mask) -{ - struct cpuset *cs; /* current cpuset ancestors */ - int allowed; /* is allocation in zone z allowed? */ - unsigned long flags; - - if (in_interrupt()) - return true; - if (node_isset(node, current->mems_allowed)) - return true; - /* - * Allow tasks that have access to memory reserves because they have - * been OOM killed to get memory anywhere. - */ - if (unlikely(test_thread_flag(TIF_MEMDIE))) - return true; - if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */ - return false; - - if (current->flags & PF_EXITING) /* Let dying task have memory */ - return true; - - /* Not hardwall and node outside mems_allowed: scan up cpusets */ - spin_lock_irqsave(&callback_lock, flags); - - rcu_read_lock(); - cs = nearest_hardwall_ancestor(task_cs(current)); - allowed = node_isset(node, cs->mems_allowed); - rcu_read_unlock(); - - spin_unlock_irqrestore(&callback_lock, flags); - return allowed; -} - -/** - * cpuset_mem_spread_node() - On which node to begin search for a file page - * cpuset_slab_spread_node() - On which node to begin search for a slab page - * - * If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for - * tasks in a cpuset with is_spread_page or is_spread_slab set), - * and if the memory allocation used cpuset_mem_spread_node() - * to determine on which node to start looking, as it will for - * certain page cache or slab cache pages such as used for file - * system buffers and inode caches, then instead of starting on the - * local node to look for a free page, rather spread the starting - * node around the tasks mems_allowed nodes. - * - * We don't have to worry about the returned node being offline - * because "it can't happen", and even if it did, it would be ok. - * - * The routines calling guarantee_online_mems() are careful to - * only set nodes in task->mems_allowed that are online. So it - * should not be possible for the following code to return an - * offline node. But if it did, that would be ok, as this routine - * is not returning the node where the allocation must be, only - * the node where the search should start. The zonelist passed to - * __alloc_pages() will include all nodes. If the slab allocator - * is passed an offline node, it will fall back to the local node. - * See kmem_cache_alloc_node(). - */ - -static int cpuset_spread_node(int *rotor) -{ - return *rotor = next_node_in(*rotor, current->mems_allowed); -} - -int cpuset_mem_spread_node(void) -{ - if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE) - current->cpuset_mem_spread_rotor = - node_random(¤t->mems_allowed); - - return cpuset_spread_node(¤t->cpuset_mem_spread_rotor); -} - -int cpuset_slab_spread_node(void) -{ - if (current->cpuset_slab_spread_rotor == NUMA_NO_NODE) - current->cpuset_slab_spread_rotor = - node_random(¤t->mems_allowed); - - return cpuset_spread_node(¤t->cpuset_slab_spread_rotor); -} - -EXPORT_SYMBOL_GPL(cpuset_mem_spread_node); - -/** - * cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's? - * @tsk1: pointer to task_struct of some task. - * @tsk2: pointer to task_struct of some other task. - * - * Description: Return true if @tsk1's mems_allowed intersects the - * mems_allowed of @tsk2. Used by the OOM killer to determine if - * one of the task's memory usage might impact the memory available - * to the other. - **/ - -int cpuset_mems_allowed_intersects(const struct task_struct *tsk1, - const struct task_struct *tsk2) -{ - return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed); -} - -/** - * cpuset_print_current_mems_allowed - prints current's cpuset and mems_allowed - * - * Description: Prints current's name, cpuset name, and cached copy of its - * mems_allowed to the kernel log. - */ -void cpuset_print_current_mems_allowed(void) -{ - struct cgroup *cgrp; - - rcu_read_lock(); - - cgrp = task_cs(current)->css.cgroup; - pr_info("%s cpuset=", current->comm); - pr_cont_cgroup_name(cgrp); - pr_cont(" mems_allowed=%*pbl\n", - nodemask_pr_args(¤t->mems_allowed)); - - rcu_read_unlock(); -} - -/* - * Collection of memory_pressure is suppressed unless - * this flag is enabled by writing "1" to the special - * cpuset file 'memory_pressure_enabled' in the root cpuset. - */ - -int cpuset_memory_pressure_enabled __read_mostly; - -/** - * cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims. - * - * Keep a running average of the rate of synchronous (direct) - * page reclaim efforts initiated by tasks in each cpuset. - * - * This represents the rate at which some task in the cpuset - * ran low on memory on all nodes it was allowed to use, and - * had to enter the kernels page reclaim code in an effort to - * create more free memory by tossing clean pages or swapping - * or writing dirty pages. - * - * Display to user space in the per-cpuset read-only file - * "memory_pressure". Value displayed is an integer - * representing the recent rate of entry into the synchronous - * (direct) page reclaim by any task attached to the cpuset. - **/ - -void __cpuset_memory_pressure_bump(void) -{ - rcu_read_lock(); - fmeter_markevent(&task_cs(current)->fmeter); - rcu_read_unlock(); -} - -#ifdef CONFIG_PROC_PID_CPUSET -/* - * proc_cpuset_show() - * - Print tasks cpuset path into seq_file. - * - Used for /proc//cpuset. - * - No need to task_lock(tsk) on this tsk->cpuset reference, as it - * doesn't really matter if tsk->cpuset changes after we read it, - * and we take cpuset_mutex, keeping cpuset_attach() from changing it - * anyway. - */ -int proc_cpuset_show(struct seq_file *m, struct pid_namespace *ns, - struct pid *pid, struct task_struct *tsk) -{ - char *buf; - struct cgroup_subsys_state *css; - int retval; - - retval = -ENOMEM; - buf = kmalloc(PATH_MAX, GFP_KERNEL); - if (!buf) - goto out; - - css = task_get_css(tsk, cpuset_cgrp_id); - retval = cgroup_path_ns(css->cgroup, buf, PATH_MAX, - current->nsproxy->cgroup_ns); - css_put(css); - if (retval >= PATH_MAX) - retval = -ENAMETOOLONG; - if (retval < 0) - goto out_free; - seq_puts(m, buf); - seq_putc(m, '\n'); - retval = 0; -out_free: - kfree(buf); -out: - return retval; -} -#endif /* CONFIG_PROC_PID_CPUSET */ - -/* Display task mems_allowed in /proc//status file. */ -void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task) -{ - seq_printf(m, "Mems_allowed:\t%*pb\n", - nodemask_pr_args(&task->mems_allowed)); - seq_printf(m, "Mems_allowed_list:\t%*pbl\n", - nodemask_pr_args(&task->mems_allowed)); -} -- cgit v1.2.3