【Linux负载系列-1】Linux 系统的平均负载（Load Avarage）

1

$ grep -c 'model name' /proc/cpuinfo

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

// kernel/sched/loadavg.c
/*
 * Global load-average calculations
 *
 * We take a distributed and async approach to calculating the global load-avg
 * in order to minimize overhead.
 *
 * The global load average is an exponentially decaying average of nr_running +
 * nr_uninterruptible.
 *
 * Once every LOAD_FREQ:
 *
 *   nr_active = 0;
 *   for_each_possible_cpu(cpu)
 *	nr_active += cpu_of(cpu)->nr_running + cpu_of(cpu)->nr_uninterruptible;
 *
 *   avenrun[n] = avenrun[0] * exp_n + nr_active * (1 - exp_n)
 *
 * Due to a number of reasons the above turns in the mess below:
 *
 *  - for_each_possible_cpu() is prohibitively expensive on machines with
 *    serious number of CPUs, therefore we need to take a distributed approach
 *    to calculating nr_active.
 *
 *        \Sum_i x_i(t) = \Sum_i x_i(t) - x_i(t_0) | x_i(t_0) := 0
 *                      = \Sum_i { \Sum_j=1 x_i(t_j) - x_i(t_j-1) }
 *
 *    So assuming nr_active := 0 when we start out -- true per definition, we
 *    can simply take per-CPU deltas and fold those into a global accumulate
 *    to obtain the same result. See calc_load_fold_active().
 *
 *    Furthermore, in order to avoid synchronizing all per-CPU delta folding
 *    across the machine, we assume 10 ticks is sufficient time for every
 *    CPU to have completed this task.
 *
 *    This places an upper-bound on the IRQ-off latency of the machine. Then
 *    again, being late doesn't loose the delta, just wrecks the sample.
 *
 *  - cpu_rq()->nr_uninterruptible isn't accurately tracked per-CPU because
 *    this would add another cross-CPU cacheline miss and atomic operation
 *    to the wakeup path. Instead we increment on whatever CPU the task ran
 *    when it went into uninterruptible state and decrement on whatever CPU
 *    did the wakeup. This means that only the sum of nr_uninterruptible over
 *    all CPUs yields the correct result.
 *
 *  This covers the NO_HZ=n code, for extra head-aches, see the comment below.
 */


long calc_load_fold_active(struct rq *this_rq, long adjust)
{
	long nr_active, delta = 0;

	nr_active = this_rq->nr_running - adjust;
	nr_active += (long)this_rq->nr_uninterruptible;

	if (nr_active != this_rq->calc_load_active) {
		delta = nr_active - this_rq->calc_load_active;
		this_rq->calc_load_active = nr_active;
	}

	return delta;
}

1
2
3
4
5
6
7

$ cat /proc/loadavg
0.30 0.54 0.59 1/2411 20645
# load1  load5 load15  nr_running/nr_threads  task_active_pid_ns
# nr_running 当前在各个 cpu 运行队列中的值， sum += foreach(cpu_rq(i)->nr_running)
# nr_threads 为 kernel/fork.c 文件中的全局变量，记录当前的整体线程数
# int nr_threads;			/* The idle threads do not count.. */
# task_active_pid_ns  这个为当前活跃的进程的 ns 对应的一个索引位置

1
2
3
4
5
6
7

#define FSHIFT		11						/* nr of bits of precision */
#define FIXED_1		(1<<FSHIFT)		/* 1.0 as fixed-point */
#define LOAD_FREQ	(5*HZ+1)			/* 5 sec intervals */   // 每 5s 更新一次

#define EXP_1		  1884					/* 1/exp(5sec/1min) as fixed-point */
#define EXP_5		  2014					/* 1/exp(5sec/5min) */
#define EXP_15		2037					/* 1/exp(5sec/15min) */

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

// From include/linux/sched/loadavg.h
#define LOAD_INT(x) ((x) >> FSHIFT)   // 右移 11 位，得到整数部分
#define LOAD_FRAC(x) LOAD_INT(((x) & (FIXED_1-1)) * 100)  // 首先获取到低的 11 位，然后乘以 100， 
																													// 再右移 11 位置，得到最高的 2 位，舍弃了低位

// 对于读取 /proc/loadavg 文件的对外输出
static int loadavg_proc_show(struct seq_file *m, void *v)
{
	unsigned long avnrun[3];

	get_avenrun(avnrun, FIXED_1/200, 0);
  // 函数运行后  avenrun[0] = avenrun[0]/(2^11) + 1/200
  // FIXED_1/200在 这里是用于小数部分第三位的四舍五入，由于小数部分只取前两位
  
  // LOAD_INT(a) = avenrun[0]/(2^11) + 1/200
	// LOAD_FRAC(a) = ((avenrun[0]%(2^11) + 2^11/200) * 100) / (2^11)
  //           = (((avenrun[0]%(2^11)) * 100 + 2^10) / (2^11)
  //           = ((avenrun[0]%(2^11) * 100) / (2^11) + 1/2

	seq_printf(m, "%lu.%02lu %lu.%02lu %lu.%02lu %ld/%d %d\n",
		LOAD_INT(avnrun[0]), LOAD_FRAC(avnrun[0]),
		LOAD_INT(avnrun[1]), LOAD_FRAC(avnrun[1]),
		LOAD_INT(avnrun[2]), LOAD_FRAC(avnrun[2]),
		nr_running(), nr_threads,
		idr_get_cursor(&task_active_pid_ns(current)->idr) - 1);
		return 0;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

/*
 * nr_running and nr_context_switches:
 *
 * externally visible scheduler statistics: current number of runnable
 * threads, total number of context switches performed since bootup.
 */
unsigned long nr_running(void)
{
	unsigned long i, sum = 0;

	for_each_online_cpu(i)
		sum += cpu_rq(i)->nr_running;

	return sum;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

// 获取负载的函数入口
/**
 * get_avenrun - get the load average array
 * @loads:	pointer to dest load array
 * @offset:	offset to add
 * @shift:	shift count to shift the result left
 *
 * These values are estimates at best, so no need for locking.
 */
void get_avenrun(unsigned long *loads, unsigned long offset, int shift)
{
	loads[0] = (avenrun[0] + offset) << shift;
	loads[1] = (avenrun[1] + offset) << shift;
	loads[2] = (avenrun[2] + offset) << shift;
}


// 计算 loadavg 的入口
/*
 * a1 = a0 * e + a * (1 - e)
 *
 * a2 = a1 * e + a * (1 - e)
 *    = (a0 * e + a * (1 - e)) * e + a * (1 - e)
 *    = a0 * e^2 + a * (1 - e) * (1 + e)
 *
 * a3 = a2 * e + a * (1 - e)
 *    = (a0 * e^2 + a * (1 - e) * (1 + e)) * e + a * (1 - e)
 *    = a0 * e^3 + a * (1 - e) * (1 + e + e^2)
 *
 *  ...
 *
 * an = a0 * e^n + a * (1 - e) * (1 + e + ... + e^n-1) [1]
 *    = a0 * e^n + a * (1 - e) * (1 - e^n)/(1 - e)
 *    = a0 * e^n + a * (1 - e^n)
 *
 * [1] application of the geometric series:
 *
 *              n         1 - x^(n+1)
 *     S_n := \Sum x^i = -------------
 *             i=0          1 - x
 */
unsigned long
calc_load_n(unsigned long load, unsigned long exp,
	    unsigned long active, unsigned int n)
{
	return calc_load(load, fixed_power_int(exp, FSHIFT, n), active);
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

/*
 * a1 = a0 * e + a * (1 - e)
 */
static inline unsigned long
calc_load(unsigned long load, unsigned long exp, unsigned long active)
{
	unsigned long newload;

	newload = load * exp + active * (FIXED_1 - exp);
	if (active >= load)
		newload += FIXED_1-1;

	return newload / FIXED_1;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

/* 
* calc_load - given tick count, update the avenrun load estimates. 
* This is called while holding a write_lock on xtime_lock. 
*/ 
static inline void calc_load(unsigned long ticks) 
{ 
        unsigned long active_tasks; /* fixed-point */ 
        static int count = LOAD_FREQ;  
        count -= ticks; 
        if (count < 0) { 
                count += LOAD_FREQ; 
                active_tasks = count_active_tasks(); 
                CALC_LOAD(avenrun[0], EXP_1, active_tasks); 
                CALC_LOAD(avenrun[1], EXP_5, active_tasks); 
                CALC_LOAD(avenrun[2], EXP_15, active_tasks); 
        } 
}
static unsigned long count_active_tasks(void) 
{ 
        return nr_active() * FIXED_1; 
}
#define LOAD_FREQ       (5*HZ)          /* 5 sec intervals */ 
#define EXP_1           1884            /* 1/exp(5sec/1min) as fixed-point */ 
#define EXP_5           2014            /* 1/exp(5sec/5min) */ 
#define EXP_15          2037            /* 1/exp(5sec/15min) */

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

unsigned long nr_active(void) 
{ 
        unsigned long i, running = 0, uninterruptible = 0; 

        for_each_online_cpu(i) { 
                running += cpu_rq(i)->nr_running; // 处于运行中的进程
                uninterruptible += cpu_rq(i)->nr_uninterruptible; // 处于uninterruptible状态的进程
        } 

        if (unlikely((long)uninterruptible < 0)) 
                uninterruptible = 0; 

        return running + uninterruptible; 
}

#define TASK_RUNNING            0 
#define TASK_INTERRUPTIBLE      1 
#define TASK_UNINTERRUPTIBLE    2 
#define TASK_STOPPED            4 
#define TASK_TRACED             8 
/* in tsk->exit_state */ 
#define EXIT_ZOMBIE             16 
#define EXIT_DEAD               32 
/* in tsk->state again */ 
#define TASK_NONINTERACTIVE     64

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

// 计算的状态包括  this_rq->nr_running +  this_rq->nr_uninterruptible
long calc_load_fold_active(struct rq *c, long adjust)
{
	long nr_active, delta = 0;

	nr_active = this_rq->nr_running - adjust;
	nr_active += (long)this_rq->nr_uninterruptible;

	if (nr_active != this_rq->calc_load_active) {
		delta = nr_active - this_rq->calc_load_active;
		this_rq->calc_load_active = nr_active;
	}

	return delta;
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

/*
 * Called from scheduler_tick() to periodically update this CPU's
 * active count.
 */
void calc_global_load_tick(struct rq *this_rq)
{
	long delta;

	if (time_before(jiffies, this_rq->calc_load_update))
		return;

	delta  = calc_load_fold_active(this_rq, 0);
	if (delta)
		atomic_long_add(delta, &calc_load_tasks);

	this_rq->calc_load_update += LOAD_FREQ;
}

1
2
3
4
5
6
7
8

/*
 * Must hold jiffies_lock
 */
void do_timer(unsigned long ticks)
{
	jiffies_64 += ticks;
	calc_global_load(ticks); // -->  calc_global_load
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

/*
 * calc_load - update the avenrun load estimates 10 ticks after the
 * CPUs have updated calc_load_tasks.
 *
 * Called from the global timer code.
 */
void calc_global_load(unsigned long ticks)
{
	unsigned long sample_window;
	long active, delta;

	sample_window = READ_ONCE(calc_load_update);
	if (time_before(jiffies, sample_window + 10)) // 检查是否到了更新窗口，负载的计算必须在计算窗口的的 10 tick 以后，
    																						// 这是为了保证所有的 CPU 的计算已经完成
		return;

	/*
	 * Fold the 'old' NO_HZ-delta to include all NO_HZ CPUs.
	 */
	delta = calc_load_nohz_read();
	if (delta)
		atomic_long_add(delta, &calc_load_tasks);

	active = atomic_long_read(&calc_load_tasks); // 读取已经更新过的 active 值
	active = active > 0 ? active * FIXED_1 : 0;

	avenrun[0] = calc_load(avenrun[0], EXP_1, active);
	avenrun[1] = calc_load(avenrun[1], EXP_5, active);
	avenrun[2] = calc_load(avenrun[2], EXP_15, active);

	WRITE_ONCE(calc_load_update, sample_window + LOAD_FREQ); // 添加为下一次更新的时间间隔  LOAD_FREQ = 5s

	/*
	 * In case we went to NO_HZ for multiple LOAD_FREQ intervals
	 * catch up in bulk.
	 */
	calc_global_nohz();
}