解析 Linux Cgroup V2 的 memory control 實現

mkdir /sys/fs/cgroup/test

ls /sys/fs/cgroup/test | grep memory
memory.current
memory.events
memory.events.local
memory.high
memory.low
memory.max
memory.min
memory.numa_stat
memory.oom.group
memory.peak
memory.pressure
memory.reclaim
memory.stat
memory.swap.current
memory.swap.events
memory.swap.high
memory.swap.max
memory.swap.peak
memory.zswap.current
memory.zswap.max
memory.zswap.writeback

// include/linux/cgroup-defs.h
struct cgroup {
	// 指向所有子系統的陣列
    struct cgroup_subsys_state *subsys[CGROUP_SUBSYS_COUNT];
}

// include/linux/memcontrol.h
// memory 子系統的實現
struct mem_cgroup {
	// 類似繼承的概念
	struct cgroup_subsys_state css;
	...
}

// include/linux/sched.h
// 行程、執行緒的物件
struct task_struct {
...
#ifdef CONFIG_CGROUPS
	/* Control Group info protected by css_set_lock: */
	// 指向到 Cgroup SubSystem SET
	struct css_set __rcu		*cgroups;
	/* cg_list protected by css_set_lock and tsk->alloc_lock: */
	struct list_head		cg_list;
#endif
...
}

// include/linux/cgroup-defs.h
struct css_set {
	/*
	 * Set of subsystem states, one for each subsystem. This array is
	 * immutable after creation apart from the init_css_set during
	 * subsystem registration (at boot time).
	 */
	struct cgroup_subsys_state *subsys[CGROUP_SUBSYS_COUNT];
...
}

// include/linux/memcontrol.h
struct mem_cgroup {
...
	struct page_counter memory;		/* Both v1 & v2 */

	union {
		struct page_counter swap;	/* v2 only */
		struct page_counter memsw;	/* v1 only */
	};
...
}

struct page_counter {
	/*
	 * Make sure 'usage' does not share cacheline with any other field in
	 * v2. The memcg->memory.usage is a hot member of struct mem_cgroup.
	 */
	atomic_long_t usage;
	unsigned long failcnt; /* v1-only field */

	CACHELINE_PADDING(_pad1_);

	/* effective memory.min and memory.min usage tracking */
	unsigned long emin;
	atomic_long_t min_usage;
	atomic_long_t children_min_usage;

	/* effective memory.low and memory.low usage tracking */
	unsigned long elow;
	atomic_long_t low_usage;
	atomic_long_t children_low_usage;

	unsigned long watermark;
	/* Latest cg2 reset watermark */
	unsigned long local_watermark;

	/* Keep all the read most fields in a separete cacheline. */
	CACHELINE_PADDING(_pad2_);

	bool protection_support;
	bool track_failcnt;
	unsigned long min;
	unsigned long low;
	unsigned long high;
	unsigned long max;
	struct page_counter *parent;
} ____cacheline_internodealigned_in_smp;

// kernel/cgroup/cgroup-v1.c
struct kernfs_syscall_ops cgroup1_kf_syscall_ops = {
	.rename			= cgroup1_rename,
	.show_options		= cgroup1_show_options,
	.mkdir			= cgroup_mkdir, // 註冊 cgroupv1 對 cgroupfs mkdir 操作的處理函數
	.rmdir			= cgroup_rmdir,
	.show_path		= cgroup_show_path,
};

// kernel/cgroup/cgroup.c
static struct kernfs_syscall_ops cgroup_kf_syscall_ops = {
	.show_options		= cgroup_show_options,
	.mkdir			= cgroup_mkdir, // 註冊 cgroupv2 對 cgroupfs mkdir 操作的處理函數
	.rmdir			= cgroup_rmdir,
	.show_path		= cgroup_show_path,
};

// cgroup/cgroup.c
int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode)
{
 ...
 // 從 cgroupfs 物件取得 cgroup 物件
 parent = cgroup_kn_lock_live(parent_kn, false);

 cgrp = cgroup_create(parent);

 cgroup_apply_control_enable(cgrp); // 初始化所有 cgroup 子系統
 ...
}

// cgroup/cgroup.c
static int cgroup_apply_control_enable(struct cgroup *cgrp)
{
 ...
 cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) {
  for_each_subsys(ss, ssid) { // iterace 每個子系統
   struct cgroup_subsys_state *css = cgroup_css(dsct, ss);
   css = css_create(dsct, ss);
   ...
  }
 }
 return 0;
}

static struct cgroup_subsys_state *css_create(struct cgroup *cgrp,
					      struct cgroup_subsys *ss)
{
	...
	css = ss->css_alloc(parent_css);
	...
	return css;
	...
}

// mm/memcontrol.c
struct cgroup_subsys memory_cgrp_subsys = {
	.css_alloc = mem_cgroup_css_alloc, // memory 子系統註冊的 css_alloc 處理函數
	.css_online = mem_cgroup_css_online,
	.css_offline = mem_cgroup_css_offline,
	.css_released = mem_cgroup_css_released,
	.css_free = mem_cgroup_css_free,
	.css_reset = mem_cgroup_css_reset,
	.css_rstat_flush = mem_cgroup_css_rstat_flush,
	.attach = mem_cgroup_attach,
	.fork = mem_cgroup_fork,
	.exit = mem_cgroup_exit,
	.dfl_cftypes = memory_files,
#ifdef CONFIG_MEMCG_V1
	.legacy_cftypes = mem_cgroup_legacy_files,
#endif
	.early_init = 0,
};

static struct cgroup_subsys_state * __ref
mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
{
	struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
	...
	memcg = mem_cgroup_alloc(parent); // 建立新的 memcg 物件
	...
	
	/* 一些初始化 */
	page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
	memcg1_soft_limit_reset(memcg);
#ifdef CONFIG_ZSWAP
	memcg->zswap_max = PAGE_COUNTER_MAX;
	WRITE_ONCE(memcg->zswap_writeback, true);
#endif
	page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
	if (parent) {
		WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent));

		// 初始化 counters
		page_counter_init(&memcg->memory, &parent->memory, memcg_on_dfl);
		page_counter_init(&memcg->swap, &parent->swap, false);
#ifdef CONFIG_MEMCG_V1
		memcg->memory.track_failcnt = !memcg_on_dfl;
		WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable));
		page_counter_init(&memcg->kmem, &parent->kmem, false);
		page_counter_init(&memcg->tcpmem, &parent->tcpmem, false);
#endif
	}
...
}

static inline void page_counter_init(struct page_counter *counter,
				     struct page_counter *parent,
				     bool protection_support)
{
	counter->usage = (atomic_long_t)ATOMIC_LONG_INIT(0);
	counter->max = PAGE_COUNTER_MAX;
	counter->parent = parent; // 保存 parent cgroup counter 的指標
	counter->protection_support = protection_support;
	counter->track_failcnt = false;
}

echo "1500" > /sys/fs/cgroup/test/memory.max

// mm/memcontrol.c
static struct cftype memory_files[] = {
...
	{
		.name = "max",
		.flags = CFTYPE_NOT_ON_ROOT, # 對 root cgroup 不顯示
		.seq_show = memory_max_show,
	},
...
}

static ssize_t memory_max_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off)
{
	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
	unsigned int nr_reclaims = MAX_RECLAIM_RETRIES; 
	bool drained = false;
	unsigned long max;
	int err;

  // 解析輸入字串
	buf = strstrip(buf);
	err = page_counter_memparse(buf, "max", &max);
	if (err)
		return err;

  // 寫入 counter
	xchg(&memcg->memory.max, max);

	...
	// 檢查現在的使用量有沒有超過新的 max，有的話觸發回收機制
	...

	memcg_wb_domain_size_changed(memcg);
	return nbytes;
}

// 將使用者輸入的 bytes 單位字串轉成 page 為單位的整數
// mm/page_counter.c
/**
 * page_counter_memparse - memparse() for page counter limits
 * @buf: string to parse
 * @max: string meaning maximum possible value
 * @nr_pages: returns the result in number of pages
 *
 * Returns -EINVAL, or 0 and @nr_pages on success.  @nr_pages will be
 * limited to %PAGE_COUNTER_MAX.
 */
int page_counter_memparse(const char *buf, const char *max,
			  unsigned long *nr_pages)
{
	char *end;
	u64 bytes;

	if (!strcmp(buf, max)) {
		*nr_pages = PAGE_COUNTER_MAX;
		return 0;
	}

	bytes = memparse(buf, &end);
	if (*end != '\0')
		return -EINVAL;

	*nr_pages = min(bytes / PAGE_SIZE, (u64)PAGE_COUNTER_MAX);

	return 0;
}

static struct folio *alloc_anon_folio(struct vm_fault *vmf) {
	...
	// 分配物理 page
	folio = vma_alloc_folio(gfp, order, vma, addr, true);
	...
	// 更新 memory cgroup
	mem_cgroup_charge(folio, vma->vm_mm, gfp)
	...
}

int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp)
{
	struct mem_cgroup *memcg;
	int ret;

	// 從 process 的 mm struct 拿到對應的 cgroup memory subsystem
	memcg = get_mem_cgroup_from_mm(mm);
	ret = charge_memcg(folio, memcg, gfp);
	css_put(&memcg->css);

	return ret;
}

static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,
			gfp_t gfp)
{
	int ret;

  // 嘗試收費
	ret = try_charge(memcg, gfp, folio_nr_pages(folio));
	if (ret)
		goto out;

	css_get(&memcg->css);
	commit_charge(folio, memcg); // folio->memcg_data = (unsigned long)memcg;
	memcg1_commit_charge(folio, memcg);
out:
	return ret;
}

static inline int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
			     unsigned int nr_pages)
{
  // root cgroup 不限制
	if (mem_cgroup_is_root(memcg))
		return 0;

	return try_charge_memcg(memcg, gfp_mask, nr_pages);
}

// mm/memcontrol.c
static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask,
			    unsigned int nr_pages)
{
	...
retry:
	// 每個 memcg 在每個 CPU 上都有一個 "stock"（庫存），預先計費了一些頁面以加速常見的小額分配。如果請求的 `nr_pages` 可以從 stock 中滿足，則計費成功
	if (consume_stock(memcg, nr_pages, gfp_mask))
		return 0;

	batch = nr_pages;

	struct page_counter *counter;
	// 除了 memory 還有 memsw 記帳 (對 swap 控制），這邊簡化掉
	// 嘗試收費 (更新所有層級的 cgroup memory counter)
	if (page_counter_try_charge(&memcg->memory, batch, &counter))
		goto done_restock; // 記帳成功
	
	// 收費失敗，有一層超過 memory.max 限制了
	// 取得超額的 cgroup
	struct mem_cgroup *mem_over_limit = mem_cgroup_from_counter(counter, memory);

	...

	// 發送 cgroup event
	memcg_memory_event(mem_over_limit, MEMCG_MAX);

	psi_memstall_enter(&pflags); // 記錄壓力停滯信息 (PSI) 開始 (memory.pressure)
	nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
	                                            gfp_mask, reclaim_options, NULL);
	                                            // 嘗試回收記憶體
	psi_memstall_leave(&pflags); // PSI 結束

	if (mem_cgroup_margin(mem_over_limit) >= nr_pages)  // 如果清出足夠的空間，在嘗試充值一次
		goto retry;

	// 將 stock 中的預留頁面返回給 cgroup，然後再試一次
	if (!drained) {
	    drain_all_stock(mem_over_limit); // 排空 mem_over_limit cgroup 的所有 CPU stock
	    drained = true;
	    goto retry;
	}

	if (gfp_mask & __GFP_NORETRY) // 如果不允許重試
	    goto nomem;
	...
	if (passed_oom && task_is_dying()) // 可能自己被殺掉了，就不處理了
		goto nomem;

	// 嘗試透過 OOM Kill 釋放作夠的記憶體
	if (mem_cgroup_oom(mem_over_limit, gfp_mask,
			   get_order(nr_pages * PAGE_SIZE))) {
		passed_oom = true;
		nr_retries = MAX_RECLAIM_RETRIES;
		goto retry;
	}
nomem:
	// 申請記憶體失敗，返回錯誤
	if (!(gfp_mask & (__GFP_NOFAIL | __GFP_HIGH)))
		return -ENOMEM;
...
force:
	// 有一些 Kernel 用的，不允許失敗，走額外邏輯
done_restock:
	// 檢查是否超出高水位線，是的話，觸發高水位回收機制
	do {
		bool mem_high, swap_high;

		mem_high = page_counter_read(&memcg->memory) >
			READ_ONCE(memcg->memory.high);
		swap_high = page_counter_read(&memcg->swap) >
			READ_ONCE(memcg->swap.high);
		...
		if (mem_high || swap_high) {
			current->memcg_nr_pages_over_high += batch;
			set_notify_resume(current);
			break;
		}
	} while ((memcg = parent_mem_cgroup(memcg)));

	if (current->memcg_nr_pages_over_high > MEMCG_CHARGE_BATCH &&
	    !(current->flags & PF_MEMALLOC) &&
	    gfpflags_allow_blocking(gfp_mask))
		mem_cgroup_handle_over_high(gfp_mask);
	return 0;
}

static bool mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
{
	bool locked, ret;

	if (order > PAGE_ALLOC_COSTLY_ORDER)
		return false;

	memcg_memory_event(memcg, MEMCG_OOM);

	// cgroup v1 特殊處理邏輯
	if (!memcg1_oom_prepare(memcg, &locked))
		return false;

	// oom 主要處理函數
	ret = mem_cgroup_out_of_memory(memcg, mask, order);

	memcg1_oom_finish(memcg, locked);

	return ret;
}

```c
bool memcg1_oom_prepare(struct mem_cgroup *memcg, bool *locked)
{
	/*
	 * cgroup1 allows disabling the OOM killer and waiting for outside
	 * handling until the charge can succeed; remember the context and put
	 * the task to sleep at the end of the page fault when all locks are
	 * released.
	 */
	 // 如果設定了 oom_kill_disable 就不執行 OOM Kill
	if (READ_ONCE(memcg->oom_kill_disable)) {
		if (current->in_user_fault) {
			css_get(&memcg->css);
			current->memcg_in_oom = memcg;
		}
		return false;
	}
	...
	return true;
}

// 對參數做一個包裝，方便在 out_of_memory 內部傳遞
static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
				     int order)
{
	struct oom_control oc = {
		.zonelist = NULL,
		.nodemask = NULL,
		.memcg = memcg,
		.gfp_mask = gfp_mask,
		.order = order,
	};
	bool ret = true;
	...
	ret = task_is_dying() || out_of_memory(&oc); // 核心處理函數

unlock:
	mutex_unlock(&oom_lock);
	return ret;
}

// mm/oom_kill.c
// OOM 除了 cgroup 限制超過外，也可能是真的物理記憶體不足等情況
// 這邊省略掉，只留下處理超過 cgroup 限制
bool out_of_memory(struct oom_control *oc)
{
	unsigned long freed = 0;

	// kernel option oom_killer_disabled 不允許 OOM KILL
	if (oom_killer_disabled)
		return false;
		
	...

	// 如果當前 process 接收到 SIGKILL 或正在退出，就當作殺死他
	/*
	 * If current has a pending SIGKILL or is exiting, then automatically
	 * select it.  The goal is to allow it to allocate so that it may
	 * quickly exit and free its memory.
	 */
	if (task_will_free_mem(current)) {
		mark_oom_victim(current);
		queue_oom_reaper(current);
		return true;
	}

	...

	// sysctl 參數啟用，在 oom 時直接 kernel panic
	check_panic_on_oom(oc);

	// 滿足條件的話，總是優先把自己殺掉
	// 不是 cgroup oom，不討論
	if (!is_memcg_oom(oc) ...
	}

	// 找到最壞的 process
	select_bad_process(oc);
	...
	// 找到合適的 process 就把它殺掉
	if (oc->chosen && oc->chosen != (void *)-1UL)
		oom_kill_process(oc, !is_memcg_oom(oc) ? "Out of memory" :
				 "Memory cgroup out of memory");
	return !!oc->chosen;
}

long oom_badness(struct task_struct *p, unsigned long totalpages)
{
	long points;
	long adj;

	/*
	 * Do not even consider tasks which are explicitly marked oom
	 * unkillable or have been already oom reaped or the are in
	 * the middle of vfork
	 */
	// 透過 /proc/[pid]/oom_score_adj 設置
	// 如果設置為 -1000 則直接跳過
	adj = (long)p->signal->oom_score_adj;
	if (adj == OOM_SCORE_ADJ_MIN ||
			test_bit(MMF_OOM_SKIP, &p->mm->flags) ||
			in_vfork(p)) {
		task_unlock(p);
		return LONG_MIN;
	}

	/*
	 * The baseline for the badness score is the proportion of RAM that each
	 * task's rss, pagetable and swap space use.
	 */
	points = get_mm_rss(p->mm) + get_mm_counter(p->mm, MM_SWAPENTS) +
		mm_pgtables_bytes(p->mm) / PAGE_SIZE;
	task_unlock(p);

	/* Normalize to oom_score_adj units */
	adj *= totalpages / 1000;
	points += adj;

	return points;
}

/*
 * Simple selection loop. We choose the process with the highest number of
 * 'points'. In case scan was aborted, oc->chosen is set to -1.
 */
// 看每個 process 的 badness 分數 (oom_evaluate_task)，找最壞的
static void select_bad_process(struct oom_control *oc)
{
	oc->chosen_points = LONG_MIN;

	if (is_memcg_oom(oc))
		mem_cgroup_scan_tasks(oc->memcg, oom_evaluate_task, oc);
	else {
		...
	}
}

static int oom_evaluate_task(struct task_struct *task, void *arg)
{
	struct oom_control *oc = arg;
	long points;

	// 不允許殺 init 跟 kernel thread
	if (oom_unkillable_task(task))
		goto next;

	...

	// 計算 badness
	points = oom_badness(task, oc->totalpages);
	// 如果出來是 LONG_MIN 則完全不考慮
	if (points == LONG_MIN || points < oc->chosen_points)
		goto next;

select:
	if (oc->chosen)
		put_task_struct(oc->chosen);
	get_task_struct(task);
	oc->chosen = task;
	oc->chosen_points = points;
next:
	return 0;
abort:
	if (oc->chosen)
		put_task_struct(oc->chosen);
	oc->chosen = (void *)-1UL;
	return 1;
}

alloc_anon_folio: return NULL
-> mem_cgroup_charge: return -ENOMEM
-> __mem_cgroup_charge: return -ENOMEM
-> try_charge_memcg: return -ENOMEM

static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
{
	...
	/* Returns NULL on OOM or ERR_PTR(-EAGAIN) if we must retry the fault */
	folio = alloc_anon_folio(vmf);
	if (IS_ERR(folio))
		return 0;
	if (!folio)
		goto oom;
	...
	
	oom:
		return VM_FAULT_OOM;
}

// arch/x86/mm/fault.c
exc_page_fault // 中斷
-> handle_page_fault // 中斷處理函數，判段存取地址屬於 kernel 還是 userspace
-> do_user_addr_fault // 處理 userspace 缺頁中斷
-> handle_mm_fault // mm/memory.c
-> __handle_mm_fault
-> handle_pte_fault
-> do_pte_missing // 判斷缺失的 pte page 類型，呼叫對應的分配函數
-> do_anonymous_page // 分配匿名 page
-> alloc_anon_folio // 分配 folio

static inline
void do_user_addr_fault(structpt_regs *regs,
			unsigned longerror_code,
			unsigned longaddress)
{
	...
	fault = handle_mm_fault(vma, address, flags | FAULT_FLAG_VMA_LOCK, regs);
	...
	if (fault & VM_FAULT_OOM) {
		/* Kernel mode? Handle exceptions or die: */
		if (!user_mode(regs)) {
			kernelmode_fixup_or_oops(regs, error_code, address,
						 SIGSEGV, SEGV_MAPERR,
						 ARCH_DEFAULT_PKEY);
			return;
		}

		/*
		 * We ran out of memory, call the OOM killer, and return the
		 * userspace (which will retry the fault, or kill us if we got
		 * oom-killed):
		 */
		pagefault_out_of_memory();
	}
	...
}

void pagefault_out_of_memory(void)
{
	static DEFINE_RATELIMIT_STATE(pfoom_rs, DEFAULT_RATELIMIT_INTERVAL,
				      DEFAULT_RATELIMIT_BURST);

	if (mem_cgroup_oom_synchronize(true))
		return;

	if (fatal_signal_pending(current))
		return;

	if (__ratelimit(&pfoom_rs))
		pr_warn("Huh VM_FAULT_OOM leaked out to the #PF handler. Retrying PF\n");
}

static inline bool mem_cgroup_oom_synchronize(bool wait)
{
	return false;
}

// mm/memcontrol-v1.c
bool mem_cgroup_oom_synchronize(bool handle)
{
	struct mem_cgroup *memcg = current->memcg_in_oom;
	struct oom_wait_info owait;
	bool locked;

	/* OOM is global, do not handle */
	if (!memcg)
		return false;

	if (!handle)
		goto cleanup;

	owait.memcg = memcg;
	owait.wait.flags = 0;
	owait.wait.func = memcg_oom_wake_function;
	owait.wait.private = current;
	INIT_LIST_HEAD(&owait.wait.entry);

	prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
	mem_cgroup_mark_under_oom(memcg);

	locked = mem_cgroup_oom_trylock(memcg);

	if (locked)
		mem_cgroup_oom_notify(memcg);

	schedule(); // 調度其他 processs
	mem_cgroup_unmark_under_oom(memcg);
	finish_wait(&memcg_oom_waitq, &owait.wait);

	if (locked)
		mem_cgroup_oom_unlock(memcg);
cleanup:
	current->memcg_in_oom = NULL;
	css_put(&memcg->css);
	return true;
}

resources:
  limits:
    cpu: 100m
    memory: 128Mi
  requests:
    cpu: 100m
    memory: 128Mi

// kubernetes/pkg/kubelet/qos/policy.go

const (
	// KubeletOOMScoreAdj is the OOM score adjustment for Kubelet
	KubeletOOMScoreAdj int = -999
	// KubeProxyOOMScoreAdj is the OOM score adjustment for kube-proxy
	KubeProxyOOMScoreAdj  int = -999
	guaranteedOOMScoreAdj int = -997
	besteffortOOMScoreAdj int = 1000
)