drivers/gator/gator_trace_sched.c - manifest_repos/kernel - Git at Google

 /**
  * Copyright (C) Arm Limited 2010-2016. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  */

 #include <trace/events/sched.h>
 #include <trace/events/task.h>

 #include "gator.h"

 #define TASK_MAP_ENTRIES        1024    /* must be power of 2 */
 #define TASK_MAX_COLLISIONS     2

 enum {
     STATE_WAIT_ON_OTHER = 0,
     STATE_CONTENTION,
     STATE_WAIT_ON_IO,
     CPU_WAIT_TOTAL
 };

 static DEFINE_PER_CPU(uint64_t *, taskname_keys);
 static DEFINE_PER_CPU(int, collecting);

 /* this array is never read as the cpu charts are derived
  * counters the files are needed, nonetheless, to show that these
  * counters are available
  */
 static const char *sched_wait_event_names[] = {
     "Linux_cpu_wait_contention",
     "Linux_cpu_wait_io",
 };
 static ulong sched_wait_enabled[ARRAY_SIZE(sched_wait_event_names)];
 static ulong sched_wait_keys[ARRAY_SIZE(sched_wait_event_names)];

 static const char *sched_activity_event_names[] = {
     "system",
     "user",
 };
 static ulong sched_activity_enabled[ARRAY_SIZE(sched_activity_event_names)][GATOR_CLUSTER_COUNT];
 static ulong sched_activity_keys[ARRAY_SIZE(sched_activity_event_names)][GATOR_CLUSTER_COUNT];

 static int sched_trace_create_files(struct super_block *sb, struct dentry *root)
 {
     struct dentry *dir;
     int i;
     int j;
     char buf[40];

     for (i = 0; i < ARRAY_SIZE(sched_wait_event_names); ++i) {
         dir = gatorfs_mkdir(sb, root, sched_wait_event_names[i]);
         if (!dir)
             return -1;
         gatorfs_create_ulong(sb, dir, "enabled", &sched_wait_enabled[i]);
         gatorfs_create_ro_ulong(sb, dir, "key", &sched_wait_keys[i]);
     }

     for (i = 0; i < ARRAY_SIZE(sched_activity_event_names); ++i) {
         for (j = 0; j < gator_cluster_count; j++) {
             snprintf(buf, sizeof(buf), "%s_%s", gator_clusters[j]->pmnc_name, sched_activity_event_names[i]);
             dir = gatorfs_mkdir(sb, root, buf);
             if (!dir)
                 return -1;
             gatorfs_create_ulong(sb, dir, "enabled", &sched_activity_enabled[i][j]);
             gatorfs_create_ro_ulong(sb, dir, "key", &sched_activity_keys[i][j]);
         }
     }

     return 0;
 }

 static void emit_pid_name(const char *comm, struct task_struct *task)
 {
     bool found = false;
     char taskcomm[TASK_COMM_LEN + 3];
     unsigned long x, cpu = get_physical_cpu();
     uint64_t *keys = &(per_cpu(taskname_keys, cpu)[(task->pid & 0xFF) * TASK_MAX_COLLISIONS]);
     uint64_t value;

     value = gator_chksum_crc32(comm);
     value = (value << 32) | (uint32_t)task->pid;

     /* determine if the thread name was emitted already */
     for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
         if (keys[x] == value) {
             found = true;
             break;
         }
     }

     if (!found) {
         /* shift values, new value always in front */
         uint64_t oldv, newv = value;

         for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
             oldv = keys[x];
             keys[x] = newv;
             newv = oldv;
         }

         /* emit pid names, cannot use get_task_comm, as it's not exported on all kernel versions */
         if (strlcpy(taskcomm, comm, TASK_COMM_LEN) == TASK_COMM_LEN - 1) {
             /* append ellipses if comm has length of TASK_COMM_LEN - 1 */
             strcat(taskcomm, "...");
         }

         marshal_thread_name(task->pid, taskcomm);
     }
 }

 static void collect_counters(u64 time, struct task_struct *task, bool sched_switch)
 {
     int *buffer, len, cpu = get_physical_cpu();
     long long *buffer64;
     struct gator_interface *gi;

     if (marshal_event_header(time)) {
         list_for_each_entry(gi, &gator_events, list) {
             if (gi->read) {
                 len = gi->read(&buffer, sched_switch);
                 if (len < 0)
                     pr_err("gator: read failed for %s\n", gi->name);
                 marshal_event(len, buffer);
             } else if (gi->read64) {
                 len = gi->read64(&buffer64, sched_switch);
                 if (len < 0)
                     pr_err("gator: read64 failed for %s\n", gi->name);
                 marshal_event64(len, buffer64);
             }
             if (gi->read_proc && task != NULL) {
                 len = gi->read_proc(&buffer64, task);
                 if (len < 0)
                     pr_err("gator: read_proc failed for %s\n", gi->name);
                 marshal_event64(len, buffer64);
             }
         }
         if (cpu == 0)
             gator_emit_perf_time(time);
         /* Only check after writing all counters so that time and corresponding counters appear in the same frame */
         buffer_check(cpu, BLOCK_COUNTER_BUF, time);

         /* Commit buffers on timeout */
         if (gator_live_rate > 0 && time >= per_cpu(gator_buffer_commit_time, cpu)) {
             static const int buftypes[] = { NAME_BUF, COUNTER_BUF, BLOCK_COUNTER_BUF, SCHED_TRACE_BUF, ACTIVITY_BUF };
             int i;

             for (i = 0; i < ARRAY_SIZE(buftypes); ++i)
                 gator_commit_buffer(cpu, buftypes[i], time);

             /* spinlocks are noops on uniprocessor machines and mutexes do
              * not work in sched_switch context in RT-Preempt full, so
              * disable proactive flushing of the annotate frame on
              * uniprocessor machines.
              */
 #ifdef CONFIG_SMP
             /* Try to preemptively flush the annotate buffer to reduce the chance of the buffer being full */
             if (on_primary_core() && spin_trylock(&annotate_lock)) {
                 gator_commit_buffer(0, ANNOTATE_BUF, time);
                 spin_unlock(&annotate_lock);
             }
 #endif
         }
     }
 }

 /* special case used during a suspend of the system */
 static void trace_sched_insert_idle(void)
 {
     marshal_sched_trace_switch(0, 0);
 }

 static void gator_trace_emit_link(struct task_struct *p)
 {
     int cookie;
     int cpu = get_physical_cpu();

     cookie = get_exec_cookie(cpu, p);
     emit_pid_name(p->comm, p);

     marshal_link(cookie, p->tgid, p->pid);
 }

 GATOR_DEFINE_PROBE(sched_process_fork, TP_PROTO(struct task_struct *parent, struct task_struct *child))
 {
     gator_trace_emit_link(child);
 }

 GATOR_DEFINE_PROBE(sched_process_exec, TP_PROTO(struct task_struct *p, pid_t old_pid, struct linux_binprm *bprm))
 {
     gator_trace_emit_link(p);
 }

 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0)
 GATOR_DEFINE_PROBE(task_rename, TP_PROTO(struct task_struct *task, char *comm))
 #else
 GATOR_DEFINE_PROBE(task_rename, TP_PROTO(struct task_struct *task, const char *comm))
 #endif
 {
     emit_pid_name(comm, task);
 }

 #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0)
 GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next))
 #else
 GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(bool preempt, struct task_struct *prev, struct task_struct *next))
 #endif
 {
     int state;
     int cpu = get_physical_cpu();

     per_cpu(in_scheduler_context, cpu) = true;

     /* do as much work as possible before disabling interrupts */
     if (prev->state == TASK_RUNNING)
         state = STATE_CONTENTION;
     else if (prev->in_iowait)
         state = STATE_WAIT_ON_IO;
     else
         state = STATE_WAIT_ON_OTHER;

     per_cpu(collecting, cpu) = 1;
     collect_counters(gator_get_time(), prev, true);
     per_cpu(collecting, cpu) = 0;

     marshal_sched_trace_switch(next->pid, state);

     per_cpu(in_scheduler_context, cpu) = false;
 }

 GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p))
 {
     marshal_sched_trace_exit(p->tgid, p->pid);
 }

 static void do_nothing(void *info)
 {
     /* Intentionally do nothing */
     (void)info;
 }

 static int register_scheduler_tracepoints(void)
 {
     /* register tracepoints */
     if (GATOR_REGISTER_TRACE(sched_process_fork))
         goto fail_sched_process_fork;
     if (GATOR_REGISTER_TRACE(sched_process_exec))
         goto fail_sched_process_exec;
     if (GATOR_REGISTER_TRACE(task_rename))
         goto fail_task_rename;
     if (GATOR_REGISTER_TRACE(sched_switch))
         goto fail_sched_switch;
     if (GATOR_REGISTER_TRACE(sched_process_free))
         goto fail_sched_process_free;
     pr_debug("gator: registered tracepoints\n");

     /* Now that the scheduler tracepoint is registered, force a context
      * switch on all cpus to capture what is currently running.
      */
     on_each_cpu(do_nothing, NULL, 0);

     return 0;

     /* unregister tracepoints on error */
 fail_sched_process_free:
     GATOR_UNREGISTER_TRACE(sched_switch);
 fail_sched_switch:
     GATOR_UNREGISTER_TRACE(task_rename);
 fail_task_rename:
     GATOR_UNREGISTER_TRACE(sched_process_exec);
 fail_sched_process_exec:
     GATOR_UNREGISTER_TRACE(sched_process_fork);
 fail_sched_process_fork:
     pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n");

     return -1;
 }

 static void unregister_scheduler_tracepoints(void)
 {
     GATOR_UNREGISTER_TRACE(sched_process_fork);
     GATOR_UNREGISTER_TRACE(sched_process_exec);
     GATOR_UNREGISTER_TRACE(task_rename);
     GATOR_UNREGISTER_TRACE(sched_switch);
     GATOR_UNREGISTER_TRACE(sched_process_free);
     pr_debug("gator: unregistered tracepoints\n");
 }

 static void gator_trace_sched_stop(void)
 {
     int cpu;

     unregister_scheduler_tracepoints();

     for_each_present_cpu(cpu) {
         kfree(per_cpu(taskname_keys, cpu));
     }
 }

 static int gator_trace_sched_start(void)
 {
     int cpu, size;
     int ret;

     for_each_present_cpu(cpu) {
         size = TASK_MAP_ENTRIES * TASK_MAX_COLLISIONS * sizeof(uint64_t);
         per_cpu(taskname_keys, cpu) = kmalloc(size, GFP_KERNEL);
         if (!per_cpu(taskname_keys, cpu))
             return -1;
         memset(per_cpu(taskname_keys, cpu), 0, size);
     }

     ret = register_scheduler_tracepoints();

     return ret;
 }

 static void gator_trace_sched_offline(void)
 {
     trace_sched_insert_idle();
 }

 static void gator_trace_sched_init(void)
 {
     int i;
     int j;

     for (i = 0; i < ARRAY_SIZE(sched_wait_enabled); i++) {
         sched_wait_enabled[i] = 0;
         sched_wait_keys[i] = gator_events_get_key();
     }

     for (i = 0; i < ARRAY_SIZE(sched_activity_enabled); i++) {
         for (j = 0; j < gator_cluster_count; j++) {
             sched_activity_enabled[i][j] = 0;
             sched_activity_keys[i][j] = gator_events_get_key();
         }
     }
 }
	/**
	* Copyright (C) Arm Limited 2010-2016. All rights reserved.
	*
	* This program is free software; you can redistribute it and/or modify
	* it under the terms of the GNU General Public License version 2 as
	* published by the Free Software Foundation.
	*
	*/

	#include <trace/events/sched.h>
	#include <trace/events/task.h>

	#include "gator.h"

	#define TASK_MAP_ENTRIES 1024 /* must be power of 2 */
	#define TASK_MAX_COLLISIONS 2

	enum {
	STATE_WAIT_ON_OTHER = 0,
	STATE_CONTENTION,
	STATE_WAIT_ON_IO,
	CPU_WAIT_TOTAL
	};

	static DEFINE_PER_CPU(uint64_t *, taskname_keys);
	static DEFINE_PER_CPU(int, collecting);

	/* this array is never read as the cpu charts are derived
	* counters the files are needed, nonetheless, to show that these
	* counters are available
	*/
	static const char *sched_wait_event_names[] = {
	"Linux_cpu_wait_contention",
	"Linux_cpu_wait_io",
	};
	static ulong sched_wait_enabled[ARRAY_SIZE(sched_wait_event_names)];
	static ulong sched_wait_keys[ARRAY_SIZE(sched_wait_event_names)];

	static const char *sched_activity_event_names[] = {
	"system",
	"user",
	};
	static ulong sched_activity_enabled[ARRAY_SIZE(sched_activity_event_names)][GATOR_CLUSTER_COUNT];
	static ulong sched_activity_keys[ARRAY_SIZE(sched_activity_event_names)][GATOR_CLUSTER_COUNT];

	static int sched_trace_create_files(struct super_block sb, struct dentry root)
	{
	struct dentry *dir;
	int i;
	int j;
	char buf[40];

	for (i = 0; i < ARRAY_SIZE(sched_wait_event_names); ++i) {
	dir = gatorfs_mkdir(sb, root, sched_wait_event_names[i]);
	if (!dir)
	return -1;
	gatorfs_create_ulong(sb, dir, "enabled", &sched_wait_enabled[i]);
	gatorfs_create_ro_ulong(sb, dir, "key", &sched_wait_keys[i]);
	}

	for (i = 0; i < ARRAY_SIZE(sched_activity_event_names); ++i) {
	for (j = 0; j < gator_cluster_count; j++) {
	snprintf(buf, sizeof(buf), "%s_%s", gator_clusters[j]->pmnc_name, sched_activity_event_names[i]);
	dir = gatorfs_mkdir(sb, root, buf);
	if (!dir)
	return -1;
	gatorfs_create_ulong(sb, dir, "enabled", &sched_activity_enabled[i][j]);
	gatorfs_create_ro_ulong(sb, dir, "key", &sched_activity_keys[i][j]);
	}
	}

	return 0;
	}

	static void emit_pid_name(const char comm, struct task_struct task)
	{
	bool found = false;
	char taskcomm[TASK_COMM_LEN + 3];
	unsigned long x, cpu = get_physical_cpu();
	uint64_t keys = &(per_cpu(taskname_keys, cpu)[(task->pid & 0xFF) TASK_MAX_COLLISIONS]);
	uint64_t value;

	value = gator_chksum_crc32(comm);
	value = (value << 32) \| (uint32_t)task->pid;

	/* determine if the thread name was emitted already */
	for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
	if (keys[x] == value) {
	found = true;
	break;
	}
	}

	if (!found) {
	/* shift values, new value always in front */
	uint64_t oldv, newv = value;

	for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
	oldv = keys[x];
	keys[x] = newv;
	newv = oldv;
	}

	/* emit pid names, cannot use get_task_comm, as it's not exported on all kernel versions */
	if (strlcpy(taskcomm, comm, TASK_COMM_LEN) == TASK_COMM_LEN - 1) {
	/* append ellipses if comm has length of TASK_COMM_LEN - 1 */
	strcat(taskcomm, "...");
	}

	marshal_thread_name(task->pid, taskcomm);
	}
	}

	static void collect_counters(u64 time, struct task_struct *task, bool sched_switch)
	{
	int *buffer, len, cpu = get_physical_cpu();
	long long *buffer64;
	struct gator_interface *gi;

	if (marshal_event_header(time)) {
	list_for_each_entry(gi, &gator_events, list) {
	if (gi->read) {
	len = gi->read(&buffer, sched_switch);
	if (len < 0)
	pr_err("gator: read failed for %s\n", gi->name);
	marshal_event(len, buffer);
	} else if (gi->read64) {
	len = gi->read64(&buffer64, sched_switch);
	if (len < 0)
	pr_err("gator: read64 failed for %s\n", gi->name);
	marshal_event64(len, buffer64);
	}
	if (gi->read_proc && task != NULL) {
	len = gi->read_proc(&buffer64, task);
	if (len < 0)
	pr_err("gator: read_proc failed for %s\n", gi->name);
	marshal_event64(len, buffer64);
	}
	}
	if (cpu == 0)
	gator_emit_perf_time(time);
	/* Only check after writing all counters so that time and corresponding counters appear in the same frame */
	buffer_check(cpu, BLOCK_COUNTER_BUF, time);

	/* Commit buffers on timeout */
	if (gator_live_rate > 0 && time >= per_cpu(gator_buffer_commit_time, cpu)) {
	static const int buftypes[] = { NAME_BUF, COUNTER_BUF, BLOCK_COUNTER_BUF, SCHED_TRACE_BUF, ACTIVITY_BUF };
	int i;

	for (i = 0; i < ARRAY_SIZE(buftypes); ++i)
	gator_commit_buffer(cpu, buftypes[i], time);

	/* spinlocks are noops on uniprocessor machines and mutexes do
	* not work in sched_switch context in RT-Preempt full, so
	* disable proactive flushing of the annotate frame on
	* uniprocessor machines.
	*/
	#ifdef CONFIG_SMP
	/* Try to preemptively flush the annotate buffer to reduce the chance of the buffer being full */
	if (on_primary_core() && spin_trylock(&annotate_lock)) {
	gator_commit_buffer(0, ANNOTATE_BUF, time);
	spin_unlock(&annotate_lock);
	}
	#endif
	}
	}
	}

	/* special case used during a suspend of the system */
	static void trace_sched_insert_idle(void)
	{
	marshal_sched_trace_switch(0, 0);
	}

	static void gator_trace_emit_link(struct task_struct *p)
	{
	int cookie;
	int cpu = get_physical_cpu();

	cookie = get_exec_cookie(cpu, p);
	emit_pid_name(p->comm, p);

	marshal_link(cookie, p->tgid, p->pid);
	}

	GATOR_DEFINE_PROBE(sched_process_fork, TP_PROTO(struct task_struct parent, struct task_struct child))
	{
	gator_trace_emit_link(child);
	}

	GATOR_DEFINE_PROBE(sched_process_exec, TP_PROTO(struct task_struct p, pid_t old_pid, struct linux_binprm bprm))
	{
	gator_trace_emit_link(p);
	}

	#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0)
	GATOR_DEFINE_PROBE(task_rename, TP_PROTO(struct task_struct task, char comm))
	#else
	GATOR_DEFINE_PROBE(task_rename, TP_PROTO(struct task_struct task, const char comm))
	#endif
	{
	emit_pid_name(comm, task);
	}

	#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0)
	GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct prev, struct task_struct next))
	#else
	GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(bool preempt, struct task_struct prev, struct task_struct next))
	#endif
	{
	int state;
	int cpu = get_physical_cpu();

	per_cpu(in_scheduler_context, cpu) = true;

	/* do as much work as possible before disabling interrupts */
	if (prev->state == TASK_RUNNING)
	state = STATE_CONTENTION;
	else if (prev->in_iowait)
	state = STATE_WAIT_ON_IO;
	else
	state = STATE_WAIT_ON_OTHER;

	per_cpu(collecting, cpu) = 1;
	collect_counters(gator_get_time(), prev, true);
	per_cpu(collecting, cpu) = 0;

	marshal_sched_trace_switch(next->pid, state);

	per_cpu(in_scheduler_context, cpu) = false;
	}

	GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p))
	{
	marshal_sched_trace_exit(p->tgid, p->pid);
	}

	static void do_nothing(void *info)
	{
	/* Intentionally do nothing */
	(void)info;
	}

	static int register_scheduler_tracepoints(void)
	{
	/* register tracepoints */
	if (GATOR_REGISTER_TRACE(sched_process_fork))
	goto fail_sched_process_fork;
	if (GATOR_REGISTER_TRACE(sched_process_exec))
	goto fail_sched_process_exec;
	if (GATOR_REGISTER_TRACE(task_rename))
	goto fail_task_rename;
	if (GATOR_REGISTER_TRACE(sched_switch))
	goto fail_sched_switch;
	if (GATOR_REGISTER_TRACE(sched_process_free))
	goto fail_sched_process_free;
	pr_debug("gator: registered tracepoints\n");

	/* Now that the scheduler tracepoint is registered, force a context
	* switch on all cpus to capture what is currently running.
	*/
	on_each_cpu(do_nothing, NULL, 0);

	return 0;

	/* unregister tracepoints on error */
	fail_sched_process_free:
	GATOR_UNREGISTER_TRACE(sched_switch);
	fail_sched_switch:
	GATOR_UNREGISTER_TRACE(task_rename);
	fail_task_rename:
	GATOR_UNREGISTER_TRACE(sched_process_exec);
	fail_sched_process_exec:
	GATOR_UNREGISTER_TRACE(sched_process_fork);
	fail_sched_process_fork:
	pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n");

	return -1;
	}

	static void unregister_scheduler_tracepoints(void)
	{
	GATOR_UNREGISTER_TRACE(sched_process_fork);
	GATOR_UNREGISTER_TRACE(sched_process_exec);
	GATOR_UNREGISTER_TRACE(task_rename);
	GATOR_UNREGISTER_TRACE(sched_switch);
	GATOR_UNREGISTER_TRACE(sched_process_free);
	pr_debug("gator: unregistered tracepoints\n");
	}

	static void gator_trace_sched_stop(void)
	{
	int cpu;

	unregister_scheduler_tracepoints();

	for_each_present_cpu(cpu) {
	kfree(per_cpu(taskname_keys, cpu));
	}
	}

	static int gator_trace_sched_start(void)
	{
	int cpu, size;
	int ret;

	for_each_present_cpu(cpu) {
	size = TASK_MAP_ENTRIES * TASK_MAX_COLLISIONS * sizeof(uint64_t);
	per_cpu(taskname_keys, cpu) = kmalloc(size, GFP_KERNEL);
	if (!per_cpu(taskname_keys, cpu))
	return -1;
	memset(per_cpu(taskname_keys, cpu), 0, size);
	}

	ret = register_scheduler_tracepoints();

	return ret;
	}

	static void gator_trace_sched_offline(void)
	{
	trace_sched_insert_idle();
	}

	static void gator_trace_sched_init(void)
	{
	int i;
	int j;

	for (i = 0; i < ARRAY_SIZE(sched_wait_enabled); i++) {
	sched_wait_enabled[i] = 0;
	sched_wait_keys[i] = gator_events_get_key();
	}

	for (i = 0; i < ARRAY_SIZE(sched_activity_enabled); i++) {
	for (j = 0; j < gator_cluster_count; j++) {
	sched_activity_enabled[i][j] = 0;
	sched_activity_keys[i][j] = gator_events_get_key();
	}
	}
	}