blob: 604ada035dbc764e2fdf75b40c7b40869fdfa778 [file] [log] [blame]
/**
* Copyright (C) ARM Limited 2010-2016. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
*/
#include "gator.h"
#include <linux/hardirq.h>
#include <linux/kthread.h>
#include <linux/sched.h>
#include <linux/semaphore.h>
#include <linux/swap.h>
#include <linux/workqueue.h>
#include <trace/events/kmem.h>
#include <linux/time.h>
#ifdef CONFIG_PREEMPT_RT_FULL
#define PREEMPT_RT_FULL 1
#else
#define PREEMPT_RT_FULL 0
#endif
#define USE_THREAD PREEMPT_RT_FULL
enum {
MEMINFO_MEMFREE,
MEMINFO_MEMUSED,
MEMINFO_BUFFERRAM,
MEMINFO_CACHED,
MEMINFO_SLAB,
MEMINFO_TOTAL,
};
enum {
PROC_SIZE,
PROC_SHARE,
PROC_TEXT,
PROC_DATA,
PROC_COUNT,
};
static const char * const meminfo_names[] = {
"Linux_meminfo_memfree",
"Linux_meminfo_memused",
"Linux_meminfo_bufferram",
"Linux_meminfo_cached",
"Linux_meminfo_slab",
};
static const char * const proc_names[] = {
"Linux_proc_statm_size",
"Linux_proc_statm_share",
"Linux_proc_statm_text",
"Linux_proc_statm_data",
};
static bool meminfo_global_enabled;
static ulong meminfo_enabled[MEMINFO_TOTAL];
static ulong meminfo_keys[MEMINFO_TOTAL];
static long long meminfo_buffer[2 * (MEMINFO_TOTAL + 2)];
static int meminfo_length;
static bool new_data_avail;
static bool proc_global_enabled;
static ulong proc_enabled[PROC_COUNT];
static ulong proc_keys[PROC_COUNT];
static DEFINE_PER_CPU(long long, proc_buffer[2 * (PROC_COUNT + 3)]);
static void do_read(void);
#if USE_THREAD
static int gator_meminfo_func(void *data);
static bool gator_meminfo_run;
/* Initialize semaphore unlocked to initialize memory values */
static DEFINE_SEMAPHORE(gator_meminfo_sem);
static void notify(void)
{
up(&gator_meminfo_sem);
}
#else
static unsigned int mem_event;
static void wq_sched_handler(struct work_struct *wsptr);
static DECLARE_WORK(work, wq_sched_handler);
static struct timer_list meminfo_wake_up_timer;
static void meminfo_wake_up_handler(unsigned long unused_data);
static void notify(void)
{
mem_event++;
}
#endif
GATOR_DEFINE_PROBE(mm_page_free, TP_PROTO(struct page *page, unsigned int order))
{
notify();
}
GATOR_DEFINE_PROBE(mm_page_free_batched, TP_PROTO(struct page *page, int cold))
{
notify();
}
GATOR_DEFINE_PROBE(mm_page_alloc, TP_PROTO(struct page *page, unsigned int order, gfp_t gfp_flags, int migratetype))
{
notify();
}
static int gator_events_meminfo_create_files(struct super_block *sb, struct dentry *root)
{
struct dentry *dir;
int i;
for (i = 0; i < MEMINFO_TOTAL; i++) {
dir = gatorfs_mkdir(sb, root, meminfo_names[i]);
if (!dir)
return -1;
gatorfs_create_ulong(sb, dir, "enabled", &meminfo_enabled[i]);
gatorfs_create_ro_ulong(sb, dir, "key", &meminfo_keys[i]);
}
for (i = 0; i < PROC_COUNT; ++i) {
dir = gatorfs_mkdir(sb, root, proc_names[i]);
if (!dir)
return -1;
gatorfs_create_ulong(sb, dir, "enabled", &proc_enabled[i]);
gatorfs_create_ro_ulong(sb, dir, "key", &proc_keys[i]);
}
return 0;
}
static int gator_events_meminfo_start(void)
{
int i;
new_data_avail = false;
meminfo_global_enabled = 0;
for (i = 0; i < MEMINFO_TOTAL; i++) {
if (meminfo_enabled[i]) {
meminfo_global_enabled = 1;
break;
}
}
proc_global_enabled = 0;
for (i = 0; i < PROC_COUNT; ++i) {
if (proc_enabled[i]) {
proc_global_enabled = 1;
break;
}
}
if (meminfo_enabled[MEMINFO_MEMUSED])
proc_global_enabled = 1;
if (meminfo_global_enabled == 0)
return 0;
if (GATOR_REGISTER_TRACE(mm_page_free))
goto mm_page_free_exit;
if (GATOR_REGISTER_TRACE(mm_page_free_batched))
goto mm_page_free_batched_exit;
if (GATOR_REGISTER_TRACE(mm_page_alloc))
goto mm_page_alloc_exit;
do_read();
#if USE_THREAD
/* Start worker thread */
gator_meminfo_run = true;
/* Since the mutex starts unlocked, memory values will be initialized */
if (IS_ERR(kthread_run(gator_meminfo_func, NULL, "gator_meminfo")))
goto kthread_run_exit;
#else
timer_setup_on_stack(&meminfo_wake_up_timer, (void*)meminfo_wake_up_handler, 0);
#endif
return 0;
#if USE_THREAD
kthread_run_exit:
GATOR_UNREGISTER_TRACE(mm_page_alloc);
#endif
mm_page_alloc_exit:
GATOR_UNREGISTER_TRACE(mm_page_free_batched);
mm_page_free_batched_exit:
GATOR_UNREGISTER_TRACE(mm_page_free);
mm_page_free_exit:
return -1;
}
static void gator_events_meminfo_stop(void)
{
if (meminfo_global_enabled) {
GATOR_UNREGISTER_TRACE(mm_page_free);
GATOR_UNREGISTER_TRACE(mm_page_free_batched);
GATOR_UNREGISTER_TRACE(mm_page_alloc);
#if USE_THREAD
/* Stop worker thread */
gator_meminfo_run = false;
up(&gator_meminfo_sem);
#else
del_timer_sync(&meminfo_wake_up_timer);
#endif
}
}
static void do_read(void)
{
struct sysinfo info;
int i, len;
unsigned long long value;
meminfo_length = len = 0;
si_meminfo(&info);
for (i = 0; i < MEMINFO_TOTAL; i++) {
if (meminfo_enabled[i]) {
switch (i) {
case MEMINFO_MEMFREE:
value = info.freeram * PAGE_SIZE;
break;
case MEMINFO_MEMUSED:
/* pid -1 means system wide */
meminfo_buffer[len++] = 1;
meminfo_buffer[len++] = -1;
/* Emit value */
meminfo_buffer[len++] = meminfo_keys[MEMINFO_MEMUSED];
meminfo_buffer[len++] = (info.totalram - info.freeram) * PAGE_SIZE;
/* Clear pid */
meminfo_buffer[len++] = 1;
meminfo_buffer[len++] = 0;
continue;
case MEMINFO_BUFFERRAM:
value = info.bufferram * PAGE_SIZE;
break;
case MEMINFO_CACHED:
// total_swapcache_pages is not exported so the result is slightly different, but hopefully not too much
value = (global_node_page_state(NR_FILE_PAGES) /*- total_swapcache_pages()*/ - info.bufferram) * PAGE_SIZE;
break;
case MEMINFO_SLAB:
value = (global_node_page_state(NR_SLAB_RECLAIMABLE) + global_node_page_state(NR_SLAB_UNRECLAIMABLE)) * PAGE_SIZE;
break;
default:
value = 0;
break;
}
meminfo_buffer[len++] = meminfo_keys[i];
meminfo_buffer[len++] = value;
}
}
meminfo_length = len;
new_data_avail = true;
}
#if USE_THREAD
static int gator_meminfo_func(void *data)
{
for (;;) {
if (down_killable(&gator_meminfo_sem))
break;
/* Eat up any pending events */
while (!down_trylock(&gator_meminfo_sem))
;
if (!gator_meminfo_run)
break;
do_read();
}
return 0;
}
#else
/* Must be run in process context as the kernel function si_meminfo() can sleep */
static void wq_sched_handler(struct work_struct *wsptr)
{
do_read();
}
static void meminfo_wake_up_handler(unsigned long unused_data)
{
/* had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater */
schedule_work(&work);
}
#endif
static int gator_events_meminfo_read(long long **buffer, bool sched_switch)
{
#if !USE_THREAD
static unsigned int last_mem_event;
#endif
if (!on_primary_core() || !meminfo_global_enabled)
return 0;
#if !USE_THREAD
if (last_mem_event != mem_event) {
last_mem_event = mem_event;
mod_timer(&meminfo_wake_up_timer, jiffies + 1);
}
#endif
if (!new_data_avail)
return 0;
new_data_avail = false;
if (buffer)
*buffer = meminfo_buffer;
return meminfo_length;
}
static int gator_events_meminfo_read_proc(long long **buffer, struct task_struct *task)
{
struct mm_struct *mm;
u64 share = 0;
int i;
long long value;
int len = 0;
int cpu = get_physical_cpu();
long long *buf = per_cpu(proc_buffer, cpu);
if (!proc_global_enabled)
return 0;
/* Collect the memory stats of the process instead of the thread */
if (task->group_leader != NULL)
task = task->group_leader;
/* get_task_mm/mmput is not needed in this context because the task and it's mm are required as part of the sched_switch */
mm = task->mm;
if (mm == NULL)
return 0;
/* Derived from task_statm in fs/proc/task_mmu.c */
if (meminfo_enabled[MEMINFO_MEMUSED] || proc_enabled[PROC_SHARE]) {
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)
share = get_mm_counter(mm, MM_FILEPAGES);
#else
share = get_mm_counter(mm, MM_FILEPAGES) + get_mm_counter(mm, MM_SHMEMPAGES);
#endif
}
/* key of 1 indicates a pid */
buf[len++] = 1;
buf[len++] = task->pid;
for (i = 0; i < PROC_COUNT; ++i) {
if (proc_enabled[i]) {
switch (i) {
case PROC_SIZE:
value = mm->total_vm;
break;
case PROC_SHARE:
value = share;
break;
case PROC_TEXT:
value = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> PAGE_SHIFT;
break;
case PROC_DATA:
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)
value = mm->total_vm - mm->shared_vm;
#else
value = mm->total_vm - mm->stack_vm;
#endif
break;
}
buf[len++] = proc_keys[i];
buf[len++] = value * PAGE_SIZE;
}
}
if (meminfo_enabled[MEMINFO_MEMUSED]) {
value = share + get_mm_counter(mm, MM_ANONPAGES);
/* Send resident for this pid */
buf[len++] = meminfo_keys[MEMINFO_MEMUSED];
buf[len++] = value * PAGE_SIZE;
}
/* Clear pid */
buf[len++] = 1;
buf[len++] = 0;
if (buffer)
*buffer = buf;
return len;
}
static struct gator_interface gator_events_meminfo_interface = {
.name = "meminfo",
.create_files = gator_events_meminfo_create_files,
.start = gator_events_meminfo_start,
.stop = gator_events_meminfo_stop,
.read64 = gator_events_meminfo_read,
.read_proc = gator_events_meminfo_read_proc,
};
int gator_events_meminfo_init(void)
{
int i;
meminfo_global_enabled = 0;
for (i = 0; i < MEMINFO_TOTAL; i++) {
meminfo_enabled[i] = 0;
meminfo_keys[i] = gator_events_get_key();
}
proc_global_enabled = 0;
for (i = 0; i < PROC_COUNT; ++i) {
proc_enabled[i] = 0;
proc_keys[i] = gator_events_get_key();
}
return gator_events_install(&gator_events_meminfo_interface);
}