doc/oprofile.html - nest-cam/v350/oprofile - Git at Google


            
            <?xml version="1.0" encoding="ISO-8859-1"?>

            
            <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">

            
            <html xmlns="http://www.w3.org/1999/xhtml">

            
              <head>

            
                <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1" />

            
                <title>OProfile manual</title>

            
                <meta name="generator" content="DocBook XSL Stylesheets V1.75.2" />

            
              </head>

            
              <body>

            
                <div class="book" title="OProfile manual">

            
                  <div class="titlepage">

            
                    <div>

            
                      <div>

            
                        <h1 class="title"><a id="oprofile-guide"></a>OProfile manual</h1>

            
                      </div>

            
                      <div>

            
                        <div class="authorgroup">

            
                          <div class="author">

            
                            <h3 class="author"><span class="firstname">John</span> <span class="surname">Levon</span></h3>

            
                            <div class="affiliation">

            
                              <div class="address">

            
                                <p>

            
                                  <code class="email">&lt;<a class="email" href="mailto:levon@movementarian.org">levon@movementarian.org</a>&gt;</code>

            
                                </p>

            
                              </div>

            
                            </div>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <div>

            
                        <p class="copyright">Copyright © 2000-2004 Victoria University of Manchester, John Levon and others</p>

            
                      </div>

            
                    </div>

            
                    <hr />

            
                  </div>

            
                  <div class="toc">

            
                    <p>

            
                      <b>Table of Contents</b>

            
                    </p>

            
                    <dl>

            
                      <dt>

            
                        <span class="chapter">

            
                          <a href="#introduction">1. Introduction</a>

            
                        </span>

            
                      </dt>

            
                      <dd>

            
                        <dl>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#legacy_mode">1. OProfile legacy profiling mode</a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#perf_events">2. OProfile perf_events profiling mode</a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#event_counting">3. OProfile event counting mode</a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#applications">4. Applications of OProfile</a>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <dl>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#jitsupport">4.1. Support for dynamically compiled (JIT) code</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#guestsupport">4.2. No support for virtual machine guests</a>

            
                                </span>

            
                              </dt>

            
                            </dl>

            
                          </dd>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#requirements">5. System requirements</a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#resources">6. Internet resources</a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#install">7. Installation</a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#uninstall">8. Uninstalling OProfile</a>

            
                            </span>

            
                          </dt>

            
                        </dl>

            
                      </dd>

            
                      <dt>

            
                        <span class="chapter">

            
                          <a href="#overview">2. Overview</a>

            
                        </span>

            
                      </dt>

            
                      <dd>

            
                        <dl>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#getting-started-with-operf">1. Getting started with OProfile using <span class="command"><strong>operf</strong></span></a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#getting-started-with-legacy">2. Getting started with OProfile using legacy profiling mode</a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#getting-started-with-ocount">3. Getting started with OProfile using <span class="command"><strong>ocount</strong></span></a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#eventspec">4. Specifying performance counter events</a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#tools-overview">5. Tools summary</a>

            
                            </span>

            
                          </dt>

            
                        </dl>

            
                      </dd>

            
                      <dt>

            
                        <span class="chapter">

            
                          <a href="#controlling-profiler">3. Controlling the profiler</a>

            
                        </span>

            
                      </dt>

            
                      <dd>

            
                        <dl>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#controlling-operf">1. Using <span class="command"><strong>operf</strong></span></a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#controlling-daemon">2. Using <span class="command"><strong>opcontrol</strong></span></a>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <dl>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#opcontrolexamples">2.1. Examples</a>

            
                                </span>

            
                              </dt>

            
                            </dl>

            
                          </dd>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#setup-jit">3. Setting up the JIT profiling feature</a>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <dl>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#setup-jit-jvm">3.1. JVM instrumentation</a>

            
                                </span>

            
                              </dt>

            
                            </dl>

            
                          </dd>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#oprofile-gui">4. Using <span class="command"><strong>oprof_start</strong></span></a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#detailed-parameters">5. Configuration details</a>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <dl>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#hardware-counters">5.1. Hardware performance counters</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#timer">5.2. OProfile in timer interrupt mode</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#p4">5.3. Pentium 4 support</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#ia64">5.4. Intel Itanium 2 support</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#ppc64">5.5. PowerPC64 support</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#cell-be">5.6. Cell Broadband Engine support</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#amd-ibs-support">5.7. AMD64 (x86_64) Instruction-Based Sampling (IBS) support</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#systemz">5.8. IBM System z hardware sampling support</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#misuse">5.9. Dangerous counter settings</a>

            
                                </span>

            
                              </dt>

            
                            </dl>

            
                          </dd>

            
                        </dl>

            
                      </dd>

            
                      <dt>

            
                        <span class="chapter">

            
                          <a href="#results">4. Obtaining profiling results</a>

            
                        </span>

            
                      </dt>

            
                      <dd>

            
                        <dl>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#profile-spec">1. Profile specifications</a>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <dl>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#profile-spec-examples">1.1. Examples</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#profile-spec-details">1.2. Profile specification parameters</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#locating-and-managing-binary-images">1.3. Locating and managing binary images</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#no-results">1.4. What to do when you don't get any results</a>

            
                                </span>

            
                              </dt>

            
                            </dl>

            
                          </dd>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#opreport">2. Image summaries and symbol summaries (<span class="command"><strong>opreport</strong></span>)</a>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <dl>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#opreport-merging">2.1. Merging separate profiles</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#opreport-comparison">2.2. Side-by-side multiple results</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#opreport-callgraph">2.3. Callgraph output</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#opreport-diff">2.4. Differential profiles with <span class="command"><strong>opreport</strong></span></a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#opreport-anon">2.5. Anonymous executable mappings</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#opreport-xml">2.6. XML formatted output</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#opreport-options">2.7. Options for <span class="command"><strong>opreport</strong></span></a>

            
                                </span>

            
                              </dt>

            
                            </dl>

            
                          </dd>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#opannotate">3. Outputting annotated source (<span class="command"><strong>opannotate</strong></span>)</a>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <dl>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#opannotate-finding-source">3.1. Locating source files</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#opannotate-details">3.2. Usage of <span class="command"><strong>opannotate</strong></span></a>

            
                                </span>

            
                              </dt>

            
                            </dl>

            
                          </dd>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#getting-jit-reports">4. OProfile results with JIT samples</a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#opgprof">5. <span class="command"><strong>gprof</strong></span>-compatible output (<span class="command"><strong>opgprof</strong></span>)</a>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <dl>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#opgprof-details">5.1. Usage of <span class="command"><strong>opgprof</strong></span></a>

            
                                </span>

            
                              </dt>

            
                            </dl>

            
                          </dd>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#oparchive">6. Analyzing profile data on another system (<span class="command"><strong>oparchive</strong></span>)</a>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <dl>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#oparchive-details">6.1. Usage of <span class="command"><strong>oparchive</strong></span></a>

            
                                </span>

            
                              </dt>

            
                            </dl>

            
                          </dd>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#opimport">7. Converting sample database files (<span class="command"><strong>opimport</strong></span>)</a>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <dl>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#opimport-details">7.1. Usage of <span class="command"><strong>opimport</strong></span></a>

            
                                </span>

            
                              </dt>

            
                            </dl>

            
                          </dd>

            
                        </dl>

            
                      </dd>

            
                      <dt>

            
                        <span class="chapter">

            
                          <a href="#interpreting">5. Interpreting profiling results</a>

            
                        </span>

            
                      </dt>

            
                      <dd>

            
                        <dl>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#irq-latency">1. Profiling interrupt latency</a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#kernel-profiling">2. Kernel profiling</a>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <dl>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#irq-masking">2.1. Interrupt masking</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#idle">2.2. Idle time</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#kernel-modules">2.3. Profiling kernel modules</a>

            
                                </span>

            
                              </dt>

            
                            </dl>

            
                          </dd>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#interpreting-callgraph">3. Interpreting call-graph profiles</a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#debug-info">4. Inaccuracies in annotated source</a>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <dl>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#effect-of-optimizations">4.1. Side effects of optimizations</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#prologues">4.2. Prologues and epilogues</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#inlined-function">4.3. Inlined functions</a>

            
                                </span>

            
                              </dt>

            
                              <dt>

            
                                <span class="sect2">

            
                                  <a href="#wrong-linenr-info">4.4. Inaccuracy in line number information</a>

            
                                </span>

            
                              </dt>

            
                            </dl>

            
                          </dd>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#symbol-without-debug-info">5. Assembly functions</a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#overlapping-symbols">6. Overlapping symbols in JITed code</a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#interpreting_operf_results">7. Using operf to profile fork/execs</a>

            
                            </span>

            
                          </dt>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#hidden-cost">8. Other discrepancies</a>

            
                            </span>

            
                          </dt>

            
                        </dl>

            
                      </dd>

            
                      <dt>

            
                        <span class="chapter">

            
                          <a href="#controlling-counter">6. Controlling the event counter</a>

            
                        </span>

            
                      </dt>

            
                      <dd>

            
                        <dl>

            
                          <dt>

            
                            <span class="sect1">

            
                              <a href="#controlling-ocount">1. Using <span class="command"><strong>ocount</strong></span></a>

            
                            </span>

            
                          </dt>

            
                        </dl>

            
                      </dd>

            
                      <dt>

            
                        <span class="chapter">

            
                          <a href="#ack">7. Acknowledgments</a>

            
                        </span>

            
                      </dt>

            
                    </dl>

            
                  </div>

            
                  <div class="chapter" title="Chapter 1. Introduction">

            
                    <div class="titlepage">

            
                      <div>

            
                        <div>

            
                          <h2 class="title"><a id="introduction"></a>Chapter 1. Introduction</h2>

            
                        </div>

            
                      </div>

            
                    </div>

            
                    <div class="toc">

            
                      <p>

            
                        <b>Table of Contents</b>

            
                      </p>

            
                      <dl>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#legacy_mode">1. OProfile legacy profiling mode</a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#perf_events">2. OProfile perf_events profiling mode</a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#event_counting">3. OProfile event counting mode</a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#applications">4. Applications of OProfile</a>

            
                          </span>

            
                        </dt>

            
                        <dd>

            
                          <dl>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#jitsupport">4.1. Support for dynamically compiled (JIT) code</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#guestsupport">4.2. No support for virtual machine guests</a>

            
                              </span>

            
                            </dt>

            
                          </dl>

            
                        </dd>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#requirements">5. System requirements</a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#resources">6. Internet resources</a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#install">7. Installation</a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#uninstall">8. Uninstalling OProfile</a>

            
                          </span>

            
                        </dt>

            
                      </dl>

            
                    </div>

            
                    <p>

            
            This manual applies to OProfile version 0.9.9.

            
            OProfile is a set of performance monitoring tools for Linux 2.6 and higher systems, available on a number of architectures.

            
            OProfile provides the following features:

            
            </p>

            
                    <div class="itemizedlist">

            
                      <ul class="itemizedlist" type="disc">

            
                        <li class="listitem">Profiler</li>

            
                        <li class="listitem">Post-processing tools for analyzing profile data</li>

            
                        <li class="listitem">Event counter</li>

            
                      </ul>

            
                    </div>

            
                    <p>

            
            </p>

            
                    <p>

            
            OProfile is capable of monitoring native hardware events occurring in all parts of a running system, from the kernel

            
            (including modules and interrupt handlers) to shared libraries

            
            to binaries. OProfile can collect event information for the whole system in the background with very little overhead. These

            
            features make it ideal for monitoring entire systems to determine bottle necks in real-world systems.

            
            </p>

            
                    <p>

            
            Many CPUs provide "performance counters", hardware registers that can count "events"; for example,

            
            cache misses, or CPU cycles. OProfile can collect profiles of code based on the number of these occurring events:

            
            repeatedly, every time a certain (configurable) number of events has occurred, the PC value is recorded.

            
            This information is aggregated into profiles for each binary image.  Alternatively, OProfile's event counting

            
            tool can collect simple raw event counts.</p>

            
                    <p>

            
            Some hardware setups do not allow OProfile to use performance counters: in these cases, no

            
            events are available so OProfile operates in timer mode, as described in later chapters. Timer

            
            mode is only available in "legacy profiling mode" (see <a class="xref" href="#legacy_mode" title="1. OProfile legacy profiling mode">Section 1, &#8220;OProfile legacy profiling mode&#8221;</a>).

            
            </p>

            
                    <div class="sect1" title="1. OProfile legacy profiling mode"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="legacy_mode"></a>1. OProfile legacy profiling mode</h2></div></div></div>

            
            "Legacy" OProfile consists of the <span class="command"><strong>opcontrol</strong></span> shell script, the <span class="command"><strong>oprofiled</strong></span> daemon, and several post-processing tools (e.g.,

            
            <span class="command"><strong>opreport</strong></span>). The <span class="command"><strong>opcontrol</strong></span> script is used for configuring, starting, and stopping a profiling session. An OProfile

            
            kernel driver (usually built as a kernel module) is used for collecting samples, which are then recorded into sample files by

            
            <span class="command"><strong>oprofiled</strong></span>. Using OProfile in "legacy mode" requires root user authority since the profiling is done on a system-wide basis, which may

            
            (if misused) cause adverse effects to the system.

            
            <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><h3 class="title">Note</h3>

            
            Profiling setup parameters that you specify using <span class="command"><strong>opcontrol</strong></span> are cached in <code class="filename">/root/.oprofile/daemonrc</code>.

            
            Subsequent runs of <code class="code">opcontrol --start</code> will continue to use these cached values until you

            
            override them with new values.

            
            </div></div>

            
                    <div class="sect1" title="2. OProfile perf_events profiling mode"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="perf_events"></a>2. OProfile perf_events profiling mode</h2></div></div></div>

            
            As of release 0.9.8, OProfile now includes the ability to profile a single process versus the system-wide technique

            
            of legacy OProfile. With this new technique, the <span class="command"><strong>operf</strong></span> program is used to control profiling instead of the

            
            <span class="command"><strong>opcontrol</strong></span> script and <span class="command"><strong>oprofiled</strong></span> daemon of leagacy mode. Also, <span class="command"><strong>operf</strong></span> does not require the

            
            special OProfile kernel driver that legacy mode does; instead, it interfaces with the kernel to collect samples via the Linux Kernel

            
            Performance Events Subsystem (hereafter referred to as "perf_events"). Using <span class="command"><strong>operf</strong></span> to profile a single

            
            process can be done as a normal user; however, root authority <span class="emphasis"><em>is</em></span> required to run <span class="command"><strong>operf</strong></span> in system-wide

            
            profiling mode.

            
            <div class="note" title="Note 1" style="margin-left: 0.5in; margin-right: 0.5in;"><h3 class="title">Note 1</h3>

            
            The same OProfile post-processing tools are used whether you collect your profile with <span class="command"><strong>operf</strong></span> or <span class="command"><strong>opcontrol</strong></span>.

            
            </div><div class="note" title="Note 2" style="margin-left: 0.5in; margin-right: 0.5in;"><h3 class="title">Note 2</h3>

            
            Some older processor models are not supported by the underlying perf_events kernel and, thus, are not supported by <span class="command"><strong>operf</strong></span>.

            
            If you receive the message

            
            <table xmlns="" border="0" style="background: #E0E0E0;" width="90%"><tr><td><pre class="screen">  Your kernel's Performance Events Subsystem does not support your processor type</pre></td></tr></table>

            
            when attempting to use <span class="command"><strong>operf</strong></span>, try profiling with <span class="command"><strong>opcontrol</strong></span>

            
            to see if your processor type may be supported by OProfile's legacy mode.

            
            </div></div>

            
                    <div class="sect1" title="3. OProfile event counting mode"><div class="titlepage"><div><div><h2 class="title" style="clear: both"><a id="event_counting"></a>3. OProfile event counting mode</h2></div></div></div>

            
            As of release 0.9.9, OProfile now includes the <span class="command"><strong>ocount</strong></span> tool which provides the capability of

            
            collecting raw event counts on a per-application, per-process, per-cpu, or system-wide basis.  Unlike the

            
            profiling tools, post-processing of the data collected is not necessary -- the data is displayed in the

            
            output of <span class="command"><strong>ocount</strong></span>.  A common use case for event counting tools is when performance analysts

            
            want to determine the CPI (cycles per instruction) for an application. High CPI implies possible stalls,

            
            and many architectures provide events that give detailed information about the different types of stalls.

            
            The events provided are architecture-specific, so we refer the reader to the hardware manuals available for

            
            the processor type being used.

            
            </div>

            
                    <div class="sect1" title="4. Applications of OProfile">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="applications"></a>4. Applications of OProfile</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            OProfile is useful in a number of situations. You might want to use OProfile when you :

            
            </p>

            
                      <div class="itemizedlist">

            
                        <ul class="itemizedlist" type="disc">

            
                          <li class="listitem">

            
                            <p>need low overhead</p>

            
                          </li>

            
                          <li class="listitem">

            
                            <p>cannot use highly intrusive profiling methods</p>

            
                          </li>

            
                          <li class="listitem">

            
                            <p>need to profile interrupt handlers</p>

            
                          </li>

            
                          <li class="listitem">

            
                            <p>need to profile an application and its shared libraries</p>

            
                          </li>

            
                          <li class="listitem">

            
                            <p>need to profile dynamically compiled code of supported virtual machines (see <a class="xref" href="#jitsupport" title="4.1. Support for dynamically compiled (JIT) code">Section 4.1, &#8220;Support for dynamically compiled (JIT) code&#8221;</a>)</p>

            
                          </li>

            
                          <li class="listitem">

            
                            <p>need to capture the performance behaviour of entire system</p>

            
                          </li>

            
                          <li class="listitem">

            
                            <p>want to examine hardware effects such as cache misses</p>

            
                          </li>

            
                          <li class="listitem">

            
                            <p>want detailed source annotation</p>

            
                          </li>

            
                          <li class="listitem">

            
                            <p>want instruction-level profiles</p>

            
                          </li>

            
                          <li class="listitem">

            
                            <p>want call-graph profiles</p>

            
                          </li>

            
                        </ul>

            
                      </div>

            
                      <p>

            
            OProfile is not a panacea. OProfile might not be a complete solution when you :

            
            </p>

            
                      <div class="itemizedlist">

            
                        <ul class="itemizedlist" type="disc">

            
                          <li class="listitem">

            
                            <p>require call graph profiles on platforms other than x86, ARM, and PowerPC</p>

            
                          </li>

            
                          <li class="listitem">

            
                            <p>require 100% instruction-accurate profiles</p>

            
                          </li>

            
                          <li class="listitem">

            
                            <p>need function call counts or an interstitial profiling API</p>

            
                          </li>

            
                          <li class="listitem">

            
                            <p>cannot tolerate any disturbance to the system whatsoever</p>

            
                          </li>

            
                          <li class="listitem">

            
                            <p>need to profile interpreted or dynamically compiled code of non-supported virtual machines</p>

            
                          </li>

            
                        </ul>

            
                      </div>

            
                      <div class="sect2" title="4.1. Support for dynamically compiled (JIT) code">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="jitsupport"></a>4.1. Support for dynamically compiled (JIT) code</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            Older versions of OProfile were not capable of attributing samples to symbols from dynamically

            
            compiled code, i.e. "just-in-time (JIT) code". Typical JIT compilers load the JIT code into

            
            anonymous memory regions. OProfile reported the samples from such code, but the attribution

            
            provided was simply:

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">     anon: &lt;tgid&gt;&lt;address range&gt;</pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            Due to this limitation, it wasn't possible to profile applications executed by virtual machines (VMs)

            
            like the Java Virtual Machine. OProfile now contains an infrastructure to support JITed code.

            
            A development library is provided to allow developers

            
            to add support for any VM that produces dynamically compiled code (see the <span class="emphasis"><em>OProfile JIT agent

            
            developer guide</em></span>).

            
            In addition, built-in support is included for the following:</p>

            
                        <div class="itemizedlist">

            
                          <ul class="itemizedlist" type="disc">

            
                            <li class="listitem">JVMTI agent library for Java (1.5 and higher)</li>

            
                            <li class="listitem">JVMPI agent library for Java (1.5 and lower)</li>

            
                          </ul>

            
                        </div>

            
                        <p>

            
            For information on how to use OProfile's JIT support, see <a class="xref" href="#setup-jit" title="3. Setting up the JIT profiling feature">Section 3, &#8220;Setting up the JIT profiling feature&#8221;</a>.

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="4.2. No support for virtual machine guests">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="guestsupport"></a>4.2. No support for virtual machine guests</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            OProfile currently does not support event-based profiling (i.e, using hardware events like cache misses,

            
            branch mispredicts) on virtual machine guests running under systems such as VMware.  The list of

            
            supported events displayed by ophelp or 'opcontrol --list-events' is based on CPU type and does

            
            not take into account whether the running system is a guest system or real system.  To use

            
            OProfile on such guest systems, you can use timer mode (see <a class="xref" href="#timer" title="5.2. OProfile in timer interrupt mode">Section 5.2, &#8220;OProfile in timer interrupt mode&#8221;</a>).

            
            </p>

            
                      </div>

            
                    </div>

            
                    <div class="sect1" title="5. System requirements">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="requirements"></a>5. System requirements</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <div class="variablelist">

            
                        <dl>

            
                          <dt>

            
                            <span class="term">Linux kernel</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			To use OProfile's JIT support, a kernel version 2.6.13 or later is required.

            
            			In earlier kernel versions, the anonymous memory regions are not reported to OProfile and results

            
            			in profiling reports without any samples in these regions.

            
            			</p>

            
                            <p>

            
                                   Profiling the Cell Broadband Engine PowerPC Processing Element (PPE) requires a kernel version

            
                                   of 2.6.18 or more recent.

            
                                   Profiling the Cell Broadband Engine Synergistic Processing Element (SPE) requires a kernel version

            
                                   of 2.6.22 or more recent.  Additionally, full support of SPE profiling requires a BFD library

            
                                   from binutils code dated January 2007 or later.  To ensure the proper BFD support exists, run

            
                                   the <code class="code">configure</code> utility with <code class="code">--with-target=cell-be</code>.


            		       Profiling the Cell Broadband Engine using SPU events requires a kernel version of 2.6.29-rc1

            
            		       or  more recent.


                                   </p>

            
                            <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><h3 class="title">Note</h3>Attempting to profile SPEs with kernel versions older than 2.6.22 may cause the

            
                                   system to crash.</div>

            
                            <p>

            
                                   </p>

            
                            <p>

            
            			Instruction-Based Sampling (IBS) profile on AMD family10h processors requires

            
            			kernel version 2.6.28-rc2 or later.

            
            			</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">Supported architecture</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			For Intel IA32, processors as old as P6 generation or Pentium 4 core are

            
            			supported.  The AMD Athlon, Opteron, Phenom, and Turion CPUs are also supported.

            
            			Older IA32 CPU types can be used with the timer mode of OProfile; please

            
            			see later in this manual for details.  OProfile also supports most processor

            
            			types of the following architectures:  Alpha, MIPS, ARM, x86-64, sparc64, PowerPC,

            
            			AVR32, and, in timer mode, PA-RISC and s390.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">Uniprocessor or SMP</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			SMP machines are fully supported.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">Required libraries</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			These libraries are required : <code class="filename">popt</code>, <code class="filename">bfd</code>,

            
            			<code class="filename">liberty</code> (debian users: libiberty is provided in binutils-dev package), <code class="filename">dl</code>,

            
            			plus the standard C++ libraries.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">Required kernel headers</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			In order to build the perf_events-enabled <span class="command"><strong>operf</strong></span> program, you need to either

            
            			install the kernel-headers package for your system or use the <code class="code">--with-kernel</code>

            
            			configure option.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">Required user account</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			For secure processing of sample data from JIT virtual machines (e.g., Java),

            
            			the special user account "oprofile" must exist on the system.  The 'configure'

            
            			and 'make install' operations will print warning messages if this

            
            			account is not found.  If you intend to profile JITed code, you must create

            
            			a group account named 'oprofile' and then create the 'oprofile' user account,

            
            			setting the default group to 'oprofile'.  A runtime error message is printed to

            
            			the oprofile log when processing JIT samples if this special user

            
            			account cannot be found.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">OProfile GUI</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			The use of the GUI to start the profiler requires the <code class="filename">Qt</code> library.

            
            			Either <code class="filename">Qt 3</code> or <code class="filename">Qt 4</code> should work.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <acronym class="acronym">ELF</acronym>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			Probably not too strenuous a requirement, but older <acronym class="acronym">A.OUT</acronym> binaries/libraries are not supported.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">K&amp;R coding style</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			OK, so it's not really a requirement, but I wish it was...

            
            		</p>

            
                          </dd>

            
                        </dl>

            
                      </div>

            
                    </div>

            
                    <div class="sect1" title="6. Internet resources">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="resources"></a>6. Internet resources</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <div class="variablelist">

            
                        <dl>

            
                          <dt>

            
                            <span class="term">Web page</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			There is a web page (which you may be reading now) at

            
            			<a class="ulink" href="http://oprofile.sf.net/">http://oprofile.sf.net/</a>.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">Download</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			You can download a source tarball or check out code from

            
            			the code repository at the sourceforge page,

            
            			<a class="ulink" href="http://sf.net/projects/oprofile/">http://sf.net/projects/oprofile/</a>.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">Mailing list</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			There is a low-traffic OProfile-specific mailing list, details at

            
            			<a class="ulink" href="http://sf.net/mail/?group_id=16191">http://sf.net/mail/?group_id=16191</a>.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">Bug tracker</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			There is a bug tracker for OProfile at SourceForge,

            
            			<a class="ulink" href="http://sf.net/tracker/?group_id=16191&amp;atid=116191">http://sf.net/tracker/?group_id=16191&amp;atid=116191</a>.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">IRC channel</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			Several OProfile developers and users sometimes hang out on channel <span class="command"><strong>#oprofile</strong></span>

            
            			on the <a class="ulink" href="http://oftc.net">OFTC</a> network.

            
            		</p>

            
                          </dd>

            
                        </dl>

            
                      </div>

            
                    </div>

            
                    <div class="sect1" title="7. Installation">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="install"></a>7. Installation</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            First you need to build OProfile and install it. <span class="command"><strong>./configure</strong></span>, <span class="command"><strong>make</strong></span>, <span class="command"><strong>make install</strong></span>

            
            is often all you need, but note these arguments to <span class="command"><strong>./configure</strong></span> :

            
            </p>

            
                      <div class="variablelist">

            
                        <dl>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--with-java</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			Use this option if you need to profile Java applications.  Also, see

            
            			<a class="xref" href="#requirements" title="5. System requirements">Section 5, &#8220;System requirements&#8221;</a>, "Required user account".  This option

            
            			is used to specify the location of the Java Development Kit (JDK)

            
            			source tree you wish to use. This is necessary to get the interface description

            
            			of the JVMPI (or JVMTI) interface to compile the JIT support code successfully.

            
            			</p>

            
                            <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;">

            
                              <h3 class="title">Note</h3>

            
                              <p>

            
            				The Java Runtime Environment (JRE) does not include the development

            
            				files that are required to compile the JIT support code, so the full

            
            				JDK must be installed in order to use this option.

            
            				</p>

            
                            </div>

            
                            <p>

            
            			By default, the Oprofile JIT support libraries will be installed in

            
            			<code class="filename">&lt;oprof_install_dir&gt;/lib/oprofile</code>.  To build

            
            			and install OProfile and the JIT support libraries as 64-bit, you can

            
            			do something like the following:

            
            			</p>

            
                            <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                              <tr>

            
                                <td>

            
                                  <pre class="screen">

            
            			# CFLAGS="-m64" CXXFLAGS="-m64" ./configure \

            
            			--with-java={my_jdk_installdir} \

            
            			--libdir=/usr/local/lib64

            
            			</pre>

            
                                </td>

            
                              </tr>

            
                            </table>

            
                            <p>

            
            			</p>

            
                            <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;">

            
                              <h3 class="title">Note</h3>

            
                              <p>

            
            				If you encounter errors building 64-bit, you should

            
            				install libtool 1.5.26 or later since that release of

            
            				libtool fixes known problems for certain platforms.

            
            				If you install libtool into a non-standard location,

            
            				you'll need to edit the invocation of 'aclocal' in

            
            				OProfile's autogen.sh as follows (assume an install

            
            				location of /usr/local):

            
            				</p>

            
                              <p>

            
            				<code class="code">aclocal -I m4 -I /usr/local/share/aclocal</code>

            
            				</p>

            
                            </div>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--with-qt-dir/includes/libraries</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			Specify the location of Qt headers and libraries. It defaults to searching in

            
            			<code class="constant">$QTDIR</code> if these are not specified.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <a id="disable-werror"></a>

            
                            <span class="term">

            
                              <code class="option">--disable-werror</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			Development versions of OProfile build by

            
            			default with <code class="option">-Werror</code>. This option turns

            
            			<code class="option">-Werror</code> off.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <a id="disable-optimization"></a>

            
                            <span class="term">

            
                              <code class="option">--disable-optimization</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			Disable the <code class="option">-O2</code> compiler flag

            
            			(useful if you discover an OProfile bug and want to give a useful

            
            			back-trace etc.)

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <a id="with-kernel"></a>

            
                            <span class="term">

            
                              <code class="option">--with-kernel</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			This option is used to specify the location of the kernel headers <code class="filename">include</code> directory

            
            			needed to build the perf_events-enabled <span class="command"><strong>operf</strong></span> program. By default, the OProfile

            
            			build system expects to find this directory under <code class="filename">/usr</code>. Use this option if your

            
            			kernel headers are in a non-standard location or if building in a cross-compile enviroment or in a

            
            			situation where the host system does not support perf_events but you wish to build binaries for a

            
            			target system that does support perf_events.

            
            		</p>

            
                          </dd>

            
                        </dl>

            
                      </div>

            
                      <p>

            
            It is recommended that if you have a

            
            uniprocessor machine, you enable the local APIC / IO_APIC support for

            
            your kernel (this is automatically enabled for SMP kernels). With many BIOS (kernel &gt;= 2.6.9 and UP kernel)

            
            it's not sufficient to enable the local APIC -- you must also turn it on explicitly at boot

            
            time by providing the "lapic" option to the kernel.

            
            If you use the NMI watchdog, be aware that the watchdog is disabled when profiling starts

            
            and not re-enabled until the profiling is stopped.

            
            </p>

            
                      <p>

            
            Please note that you must save or have available the <code class="filename">vmlinux</code> file

            
            generated during a kernel compile, as OProfile needs it (you can use

            
            <code class="option">--no-vmlinux</code>, but this will prevent kernel profiling).

            
            </p>

            
                    </div>

            
                    <div class="sect1" title="8. Uninstalling OProfile">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="uninstall"></a>8. Uninstalling OProfile</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            You must have the source tree available to uninstall OProfile; a <span class="command"><strong>make uninstall</strong></span> will

            
            remove all installed files except your configuration file in the directory <code class="filename">~/.oprofile</code>.

            
            </p>

            
                    </div>

            
                  </div>

            
                  <div class="chapter" title="Chapter 2. Overview">

            
                    <div class="titlepage">

            
                      <div>

            
                        <div>

            
                          <h2 class="title"><a id="overview"></a>Chapter 2. Overview</h2>

            
                        </div>

            
                      </div>

            
                    </div>

            
                    <div class="toc">

            
                      <p>

            
                        <b>Table of Contents</b>

            
                      </p>

            
                      <dl>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#getting-started-with-operf">1. Getting started with OProfile using <span class="command"><strong>operf</strong></span></a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#getting-started-with-legacy">2. Getting started with OProfile using legacy profiling mode</a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#getting-started-with-ocount">3. Getting started with OProfile using <span class="command"><strong>ocount</strong></span></a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#eventspec">4. Specifying performance counter events</a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#tools-overview">5. Tools summary</a>

            
                          </span>

            
                        </dt>

            
                      </dl>

            
                    </div>

            
                    <div class="sect1" title="1. Getting started with OProfile using operf">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="getting-started-with-operf"></a>1. Getting started with OProfile using <span class="command"><strong>operf</strong></span></h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            Profiling with <span class="command"><strong>operf</strong></span> is the recommended profiling mode with OProfile. Using

            
            this mode not only allows you to target your profiling more precisely (i.e., single process

            
            or system-wide), it also allows OProfile to co-exist better with other tools on your system that

            
            may also be using the perf_events kernel subsystem.

            
            </p>

            
                      <p>

            
            With <span class="command"><strong>operf</strong></span>, there is no initial setup needed -- simply invoke <span class="command"><strong>operf</strong></span> with

            
            the options you need; then run the OProfile post-processing tool(s). The <span class="command"><strong>operf</strong></span> syntax

            
            is as follows:

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">operf [ options ] [ --system-wide | --pid=&lt;PID&gt; | [ command [ args ] ] ]</pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            A typical usage might look like this:

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">operf ./my_test_program my_arg</pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            When <code class="filename">./my_test_program</code> completes (or when you press Ctrl-C), profiling

            
            stops and you're ready to use <span class="command"><strong>opreport</strong></span> or other OProfile post-processing tools.

            
            By default, <span class="command"><strong>operf</strong></span> stores the sample data in <code class="filename">&lt;cur_dir&gt;/oprofile_data/samples/current</code>,

            
            and <span class="command"><strong>opreport</strong></span> and other post-processing tools will look in that location first for profile data,

            
            unless you pass the <code class="code">--session-dir</code> option.

            
            </p>

            
                    </div>

            
                    <div class="sect1" title="2. Getting started with OProfile using legacy profiling mode">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="getting-started-with-legacy"></a>2. Getting started with OProfile using legacy profiling mode</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            Before you can use OProfile's legacy profiling mode, you must set it up. The minimum setup required for this

            
            is to tell OProfile where the <code class="filename">vmlinux</code> file corresponding to the

            
            running kernel is, for example :

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">opcontrol --vmlinux=/boot/vmlinux-`uname -r`</pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            If you don't want to profile the kernel itself,

            
            you can tell OProfile you don't have a <code class="filename">vmlinux</code> file :

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">opcontrol --no-vmlinux</pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            Now we are ready to start the daemon (<span class="command"><strong>oprofiled</strong></span>) which collects

            
            the profile data :

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">opcontrol --start</pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            When you want to stop profiling, you can do so with :

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">opcontrol --shutdown</pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            Note that unlike <span class="command"><strong>gprof</strong></span>, no instrumentation (<code class="option">-pg</code>

            
            and <code class="option">-a</code> options to <span class="command"><strong>gcc</strong></span>)

            
            is necessary.

            
            </p>

            
                      <p>

            
            Periodically (or on <span class="command"><strong>opcontrol --shutdown</strong></span> or <span class="command"><strong>opcontrol --dump</strong></span>)

            
            the profile data is written out into the $SESSION_DIR/samples directory (by default at <code class="filename">/var/lib/oprofile/samples</code>).

            
            These profile files cover shared libraries, applications, the kernel (vmlinux), and kernel modules.

            
            You can clear the profile data (at any time) with <span class="command"><strong>opcontrol --reset</strong></span>.

            
            </p>

            
                      <p>

            
            To place these sample database files in a specific directory instead of the default location

            
            (<code class="filename">/var/lib/oprofile</code>) use the <code class="option">--session-dir=dir</code> option.

            
            You must also specify the <code class="option">--session-dir</code> to tell the tools to continue using this directory.

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">opcontrol --no-vmlinux --session-dir=/home/me/tmpsession</pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">opcontrol --start --session-dir=/home/me/tmpsession</pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            You can get summaries of this data in a number of ways at any time. To get a summary of

            
            data across the entire system for all of these profiles, you can do :

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">opreport [--session-dir=dir]</pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            Or to get a more detailed summary, for a particular image, you can do something like :

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">opreport -l /boot/vmlinux-`uname -r`</pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            There are also a number of other ways of presenting the data, as described later in this manual.

            
            Note that OProfile will choose a default profiling setup for you. However, there are a number

            
            of options you can pass to <span class="command"><strong>opcontrol</strong></span> if you need to change something,

            
            also detailed later.

            
            </p>

            
                    </div>

            
                    <div class="sect1" title="3. Getting started with OProfile using ocount">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="getting-started-with-ocount"></a>3. Getting started with OProfile using <span class="command"><strong>ocount</strong></span></h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            <span class="command"><strong>ocount</strong></span> is an OProfile tool that can be used to count native hardware events occurring in either

            
            a specific application, a set of processes or threads, a set of active system processors, or the

            
            entire system. The data collected during a counting session is displayed to stdout by default, but may

            
            also be saved to a file.  The <span class="command"><strong>ocount</strong></span> syntax is as follows:

            
            </p>

            
                      <p>

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">ocount [ options ] [ --system-wide | --process-list &lt;pids&gt; | --thread-list &lt;tids&gt; | --cpu-list &lt;cpus&gt; [ command [ args ] ] ]

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            </p>

            
                      <p>

            
            A typical usage might look like this:

            
            </p>

            
                      <p>

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">ocount --events=CPU_CLK_UNHALTED,INST_RETIRED /home/user1/my_test_program my_arg</pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            </p>

            
                      <p>

            
            When <code class="filename">my_test_program</code> completes (or when you press Ctrl-C), counting

            
            stops and the results are displayed to the screen (as shown below).

            
            </p>

            
                      <p>

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            Events were actively counted for 2.8 seconds.

            
            Event counts (actual) for /home/user1/my_test_program my_arg:

            
            	Event                   Count                    % time counted

            
            	CPU_CLK_UNHALTED        9,408,018,070            100.00

            
            	INST_RETIRED            16,719,918,108           100.00

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            </p>

            
                      <p>

            
            </p>

            
                    </div>

            
                    <div class="sect1" title="4. Specifying performance counter events">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="eventspec"></a>4. Specifying performance counter events</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            Both methods of profiling (<span class="command"><strong>operf</strong></span> and <span class="command"><strong>opcontrol</strong></span>) --

            
            as well as event counting with <span class="command"><strong>ocount</strong></span> --

            
            allow you to give one or more event specifications to provide details of how each

            
            hardware performance counter should be set up. With <span class="command"><strong>operf</strong></span> and <span class="command"><strong>ocount</strong></span>, you

            
            can provide a comma-separated list of event specfications using the <code class="code">--events</code>

            
            option.  With <span class="command"><strong>opcontrol</strong></span>, you use the <code class="code">--event</code> option

            
            for each desired event specification.

            
            For profiling, the event specification is a colon-separated string of the form

            
            <code class="option"><span class="emphasis"><em>name</em></span>:<span class="emphasis"><em>count</em></span>:<span class="emphasis"><em>unitmask</em></span>:<span class="emphasis"><em>kernel</em></span>:<span class="emphasis"><em>user</em></span></code>

            
            as described in the table below. For <span class="command"><strong>ocount</strong></span>, specification is of the form

            
            <code class="option"><span class="emphasis"><em>name</em></span>:<span class="emphasis"><em>unitmask</em></span>:<span class="emphasis"><em>kernel</em></span>:<span class="emphasis"><em>user</em></span></code>.

            
            Note the presence of the <span class="emphasis"><em>count</em></span> field for profiling.  The <span class="emphasis"><em>count</em></span> field tells the profiler

            
            how many events should occur between a profile snapshot (usually referred to as a "sample").  Since

            
            <span class="command"><strong>ocount</strong></span> does not do sampling, the <span class="emphasis"><em>count</em></span> field is not needed.

            
            </p>

            
                      <p>

            
            If no event specs are passed to <span class="command"><strong>operf</strong></span>, <span class="command"><strong>ocount</strong></span>, or <span class="command"><strong>opcontrol</strong></span>,

            
            the default event will be used. With <span class="command"><strong>opcontrol</strong></span>, if you have

            
            previously specified some non-default event but want to revert to the default event, use

            
            <code class="option">--event=default</code>. Use of this option overrides all previous event selections

            
            that have been cached.

            
            </p>

            
                      <p>

            
            </p>

            
                      <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><h3 class="title">Note</h3>OProfile will allocate hardware counters as necessary, but some processor

            
            types have restrictions as to what hardware events may be counted simultaneously.

            
            <span class="command"><strong>operf</strong></span> and <span class="command"><strong>ocount</strong></span> use a multiplexing technique when such

            
            hardware restrictions are encountered, but <span class="command"><strong>opcontrol</strong></span> does

            
            not have this capability; instead, <span class="command"><strong>opcontrol</strong></span> will display an

            
            error message if you select an incompatible set of events.

            
            </div>

            
                      <p>

            
            </p>

            
                      <div class="informaltable">

            
                        <table border="1">

            
                          <colgroup>

            
                            <col />

            
                            <col />

            
                          </colgroup>

            
                          <tbody>

            
                            <tr>

            
                              <td>

            
                                <code class="option">name</code>

            
                              </td>

            
                              <td>The symbolic event name, e.g. <code class="constant">CPU_CLK_UNHALTED</code></td>

            
                            </tr>

            
                            <tr>

            
                              <td>

            
                                <code class="option">count</code>

            
                              </td>

            
                              <td>The counter reset value, e.g. 100000; use only for profiling</td>

            
                            </tr>

            
                            <tr>

            
                              <td>

            
                                <code class="option">unitmask</code>

            
                              </td>

            
                              <td>The unit mask, as given in the events list: e.g. 0x0f; or a symbolic name

            
            if a <code class="constant">name=&lt;um_name&gt;</code> field is present</td>

            
                            </tr>

            
                            <tr>

            
                              <td>

            
                                <code class="option">kernel</code>

            
                              </td>

            
                              <td>Whether to profile kernel code</td>

            
                            </tr>

            
                            <tr>

            
                              <td>

            
                                <code class="option">user</code>

            
                              </td>

            
                              <td>Whether to profile userspace code</td>

            
                            </tr>

            
                          </tbody>

            
                        </table>

            
                      </div>

            
                      <p>

            
            The last three values are optional, if you omit them (e.g. <code class="option">operf --events=DATA_MEM_REFS:30000</code>),

            
            they will be set to the default values (the default unit mask value for the given event, and profiling (or counting)

            
            both kernel and userspace code). Note that some events require a unit mask.

            
            </p>

            
                      <p>

            
            You can specify unit mask values using either a numerical value (hex values

            
            <span class="emphasis"><em>must</em></span> begin with "0x") or a symbolic name (if the <code class="constant">name=&lt;um_name&gt;</code>

            
            field is shown in the <span class="command"><strong>ophelp</strong></span> output). For some named unit masks, the hex value is not unique; thus, OProfile

            
            tools enforce specifying such unit masks value by name.

            
            </p>

            
                      <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;">

            
                        <h3 class="title">Note</h3>

            
                        <p>

            
            When using legacy mode <span class="command"><strong>opcontrol</strong></span> on IBM PowerPC platforms, all events specified must be in the same group;

            
            i.e., the group number appended to the event name (e.g. <code class="constant">&lt;<span class="emphasis"><em>some-event-name</em></span>&gt;_GRP9

            
            </code>) must be the same.

            
            </p>

            
                        <p>

            
            When using <span class="command"><strong>operf</strong></span> or <span class="command"><strong>ocount</strong></span> on IBM PowerPC platforms, the above restriction

            
            regarding the same group number does not apply, and events may be

            
            specified with or without the group number suffix. If no group number suffix is given, one will be automatically

            
            assigned; thus, OProfile post-processing tools will always show real event

            
            names that include the group number suffix.

            
            </p>

            
                      </div>

            
                      <p>

            
            If OProfile is using timer-interrupt mode, there is no event configuration possible.

            
            </p>

            
                      <p>

            
            The table below lists the default profiling event for various processor types. The same events

            
            can be used for <span class="command"><strong>ocount</strong></span>, minus the <span class="emphasis"><em>count</em></span> field.

            
            </p>

            
                      <div class="informaltable">

            
                        <table border="1">

            
                          <colgroup>

            
                            <col />

            
                            <col />

            
                            <col />

            
                          </colgroup>

            
                          <tbody>

            
                            <tr>

            
                              <td>Processor</td>

            
                              <td>cpu_type</td>

            
                              <td>Default event</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Alpha EV4</td>

            
                              <td>alpha/ev4</td>

            
                              <td>CYCLES:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Alpha EV5</td>

            
                              <td>alpha/ev5</td>

            
                              <td>CYCLES:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Alpha PCA56</td>

            
                              <td>alpha/pca56</td>

            
                              <td>CYCLES:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Alpha EV6</td>

            
                              <td>alpha/ev6</td>

            
                              <td>CYCLES:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Alpha EV67</td>

            
                              <td>alpha/ev67</td>

            
                              <td>CYCLES:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>ARM/XScale PMU1</td>

            
                              <td>arm/xscale1</td>

            
                              <td>CPU_CYCLES:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>ARM/XScale PMU2</td>

            
                              <td>arm/xscale2</td>

            
                              <td>CPU_CYCLES:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>ARM/MPCore</td>

            
                              <td>arm/mpcore</td>

            
                              <td>CPU_CYCLES:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>AVR32</td>

            
                              <td>avr32</td>

            
                              <td>CPU_CYCLES:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Athlon</td>

            
                              <td>i386/athlon</td>

            
                              <td>CPU_CLK_UNHALTED:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Pentium Pro</td>

            
                              <td>i386/ppro</td>

            
                              <td>CPU_CLK_UNHALTED:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Pentium II</td>

            
                              <td>i386/pii</td>

            
                              <td>CPU_CLK_UNHALTED:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Pentium III</td>

            
                              <td>i386/piii</td>

            
                              <td>CPU_CLK_UNHALTED:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Pentium M (P6 core)</td>

            
                              <td>i386/p6_mobile</td>

            
                              <td>CPU_CLK_UNHALTED:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Pentium 4 (non-HT)</td>

            
                              <td>i386/p4</td>

            
                              <td>GLOBAL_POWER_EVENTS:100000:1:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Pentium 4 (HT)</td>

            
                              <td>i386/p4-ht</td>

            
                              <td>GLOBAL_POWER_EVENTS:100000:1:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Hammer</td>

            
                              <td>x86-64/hammer</td>

            
                              <td>CPU_CLK_UNHALTED:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Family10h</td>

            
                              <td>x86-64/family10</td>

            
                              <td>CPU_CLK_UNHALTED:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Family11h</td>

            
                              <td>x86-64/family11h</td>

            
                              <td>CPU_CLK_UNHALTED:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Itanium</td>

            
                              <td>ia64/itanium</td>

            
                              <td>CPU_CYCLES:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>Itanium 2</td>

            
                              <td>ia64/itanium2</td>

            
                              <td>CPU_CYCLES:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>TIMER_INT</td>

            
                              <td>timer</td>

            
                              <td>None selectable</td>

            
                            </tr>

            
                            <tr>

            
                              <td>IBM pseries</td>

            
                              <td>PowerPC 4/5/6/7/8/970/Cell</td>

            
                              <td>CYCLES:100000:0:1:1</td>

            
                            </tr>

            
                            <tr>

            
                              <td>IBM s390</td>

            
                              <td>timer</td>

            
                              <td>None selectable</td>

            
                            </tr>

            
                            <tr>

            
                              <td>IBM s390x</td>

            
                              <td>timer</td>

            
                              <td>None selectable</td>

            
                            </tr>

            
                          </tbody>

            
                        </table>

            
                      </div>

            
                    </div>

            
                    <div class="sect1" title="5. Tools summary">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="tools-overview"></a>5. Tools summary</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            This section gives a brief description of the available OProfile utilities and their purpose.

            
            </p>

            
                      <div class="variablelist">

            
                        <dl>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="filename">ophelp</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This utility lists the available events and short descriptions.

            
            	</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="filename">operf</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This is the recommended program for collecting profile data, discussed in <a class="xref" href="#controlling-operf" title="1. Using operf">Section 1, &#8220;Using <span class="command"><strong>operf</strong></span>&#8221;</a>.

            
            	</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="filename">opcontrol</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Used for controlling OProfile data collection in legacy mode, discussed in <a class="xref" href="#controlling-daemon" title="2. Using opcontrol">Section 2, &#8220;Using <span class="command"><strong>opcontrol</strong></span>&#8221;</a>.

            
            	</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="filename">agent libraries</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            			Used by virtual machines (like the Java VM) to record information about JITed code being profiled. See <a class="xref" href="#setup-jit" title="3. Setting up the JIT profiling feature">Section 3, &#8220;Setting up the JIT profiling feature&#8221;</a>.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="filename">opreport</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This is the main tool for retrieving useful profile data, described in

            
            		<a class="xref" href="#opreport" title="2. Image summaries and symbol summaries (opreport)">Section 2, &#8220;Image summaries and symbol summaries (<span class="command"><strong>opreport</strong></span>)&#8221;</a>.

            
            	</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="filename">opannotate</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This utility can be used to produce annotated source, assembly or mixed source/assembly.

            
            		Source level annotation is available only if the application was compiled with

            
            		debugging symbols. See <a class="xref" href="#opannotate" title="3. Outputting annotated source (opannotate)">Section 3, &#8220;Outputting annotated source (<span class="command"><strong>opannotate</strong></span>)&#8221;</a>.

            
            	</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="filename">opgprof</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This utility can output gprof-style data files for a binary, for use with

            
            		<span class="command"><strong>gprof -p</strong></span>. See <a class="xref" href="#opgprof" title="5. gprof-compatible output (opgprof)">Section 5, &#8220;<span class="command"><strong>gprof</strong></span>-compatible output (<span class="command"><strong>opgprof</strong></span>)&#8221;</a>.

            
            	</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="filename">oparchive</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This utility can be used to collect executables, debuginfo,

            
            		and sample files and copy the files into an archive.

            
            		The archive is self-contained and can be moved to another

            
            		machine for further analysis.

            
            		See <a class="xref" href="#oparchive" title="6. Analyzing profile data on another system (oparchive)">Section 6, &#8220;Analyzing profile data on another system (<span class="command"><strong>oparchive</strong></span>)&#8221;</a>.

            
            	</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="filename">opimport</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This utility converts sample database files from a foreign binary format (abi) to

            
            		the native format. This is useful only when moving sample files between hosts,

            
            		for analysis on platforms other than the one used for collection.

            
            		See <a class="xref" href="#opimport" title="7. Converting sample database files (opimport)">Section 7, &#8220;Converting sample database files (<span class="command"><strong>opimport</strong></span>)&#8221;</a>.

            
            	</p>

            
                          </dd>

            
                        </dl>

            
                      </div>

            
                    </div>

            
                  </div>

            
                  <div class="chapter" title="Chapter 3. Controlling the profiler">

            
                    <div class="titlepage">

            
                      <div>

            
                        <div>

            
                          <h2 class="title"><a id="controlling-profiler"></a>Chapter 3. Controlling the profiler</h2>

            
                        </div>

            
                      </div>

            
                    </div>

            
                    <div class="toc">

            
                      <p>

            
                        <b>Table of Contents</b>

            
                      </p>

            
                      <dl>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#controlling-operf">1. Using <span class="command"><strong>operf</strong></span></a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#controlling-daemon">2. Using <span class="command"><strong>opcontrol</strong></span></a>

            
                          </span>

            
                        </dt>

            
                        <dd>

            
                          <dl>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#opcontrolexamples">2.1. Examples</a>

            
                              </span>

            
                            </dt>

            
                          </dl>

            
                        </dd>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#setup-jit">3. Setting up the JIT profiling feature</a>

            
                          </span>

            
                        </dt>

            
                        <dd>

            
                          <dl>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#setup-jit-jvm">3.1. JVM instrumentation</a>

            
                              </span>

            
                            </dt>

            
                          </dl>

            
                        </dd>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#oprofile-gui">4. Using <span class="command"><strong>oprof_start</strong></span></a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#detailed-parameters">5. Configuration details</a>

            
                          </span>

            
                        </dt>

            
                        <dd>

            
                          <dl>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#hardware-counters">5.1. Hardware performance counters</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#timer">5.2. OProfile in timer interrupt mode</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#p4">5.3. Pentium 4 support</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#ia64">5.4. Intel Itanium 2 support</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#ppc64">5.5. PowerPC64 support</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#cell-be">5.6. Cell Broadband Engine support</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#amd-ibs-support">5.7. AMD64 (x86_64) Instruction-Based Sampling (IBS) support</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#systemz">5.8. IBM System z hardware sampling support</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#misuse">5.9. Dangerous counter settings</a>

            
                              </span>

            
                            </dt>

            
                          </dl>

            
                        </dd>

            
                      </dl>

            
                    </div>

            
                    <div class="sect1" title="1. Using operf">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="controlling-operf"></a>1. Using <span class="command"><strong>operf</strong></span></h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            This section describes in detail how <span class="command"><strong>operf</strong></span> is used to

            
            control profiling. Unless otherwise directed, <span class="command"><strong>operf</strong></span> will profile using

            
            the default event for your system. For most systems, the default event is some

            
            cycles-based event, assuming your processor type supports hardware performance

            
            counters. If your hardware <span class="emphasis"><em>does</em></span> support performance counters, you can specify

            
            something other than the default hardware event on which to profile. The performance

            
            monitor counters can be programmed to count various hardware events,

            
            such as cache misses or MMX operations. The event

            
            chosen for each counter is reflected in the profile data collected

            
            by OProfile: functions and binaries at the top of the profiles reflect

            
            that most of the chosen events happened within that code.

            
            </p>

            
                      <p>

            
            Additionally, each counter is programmed with a "count" value, which corresponds to how

            
            detailed the profile is. The lower the value, the more frequently profile

            
            samples are taken. You can choose to sample only kernel code, user-space code,

            
            or both (both is the default). Finally, some events have a "unit mask"

            
            -- this is a value that further restricts the types of event that are counted.

            
            You can see the event types and unit masks for your CPU using <span class="command"><strong>ophelp</strong></span>.

            
            More information on event specification can be found at <a class="xref" href="#eventspec" title="4. Specifying performance counter events">Section 4, &#8220;Specifying performance counter events&#8221;</a>.

            
            </p>

            
                      <p>

            
            The <span class="command"><strong>operf</strong></span> command syntax is:

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">operf [ options ] [ --system-wide | --pid=&lt;PID&gt; | [ command [ args ] ] ]</pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            When profiling an application using either the <code class="code">command</code> or <code class="code">--pid</code> option of

            
            <span class="command"><strong>operf</strong></span>, forks and execs of the profiled process will also be profiled. The samples

            
            from an exec'ed process will be attributed to the executable binary run by that process. See

            
            <a class="xref" href="#interpreting_operf_results" title="7. Using operf to profile fork/execs">Section 7, &#8220;Using operf to profile fork/execs&#8221;</a>

            
            </p>

            
                      <p>

            
            Following is a description of the <span class="command"><strong>operf</strong></span> options.

            
            </p>

            
                      <div class="variablelist">

            
                        <dl>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">command [args]</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		The command or application to be profiled. The<span class="emphasis"><em>[args]</em></span> are the input arguments

            
                    that the command or application requires. Either <code class="code">command</code>, <code class="code">--pid</code> or

            
                    <code class="code">--system-wide</code> is required, but cannot be used simultaneously.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--pid / -p [PID]</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This option enables <span class="command"><strong>operf</strong></span> to profile a running application. <code class="code">PID</code>

            
            		should be the process ID of the process you wish to profile. When

            
            		finished profiling (e.g., when the profiled process ends), press

            
            		Ctrl-c to stop <span class="command"><strong>operf</strong></span>.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--system-wide / -s</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This option is for performing a system-wide profile. You must

            
            		have root authority to run <span class="command"><strong>operf</strong></span> in this mode.

            
            		When finished profiling, Ctrl-C to stop <span class="command"><strong>operf</strong></span>. If you run

            
            		<code class="code">operf --system-wide</code> as a background job (i.e., with the &amp;), you

            
            		<span class="emphasis"><em>must</em></span> stop it in a controlled manner in order to process

            
            		the profile data it has collected.  Use <code class="code">kill -SIGINT &lt;operf-PID&gt;</code>

            
            		for this purpose. It is recommended that when running <span class="command"><strong>operf</strong></span>

            
            		with this option, your current working directory should be <code class="filename">/root</code> or a subdirectory

            
            		of <code class="filename">/root</code> to avoid storing sample data files in locations accessible by regular users.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--vmlinux / k [vmlinux_path]</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		A vmlinux file that matches the running kernel that has symbol and/or debuginfo.

            
            		Kernel samples will be attributed to this binary, allowing post-processing tools

            
            		(like <span class="command"><strong>opreport</strong></span>) to attribute samples to the appropriate kernel symbols.

            
            		If this option is not specified, all kernel samples will be attributed to a pseudo

            
            		binary named "no-vmlinux".

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--callgraph / -g</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This option enables the callgraph to be saved during profiling. NOTE: The

            
            		full callchain is recorded, so there is no depth limit.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--append / -a</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		By default, <span class="command"><strong>operf</strong></span> moves old profile data from

            
            		<code class="filename">&lt;session_dir&gt;/samples/current</code> to

            
            		<code class="filename">&lt;session_dir&gt;/samples/previous</code>.

            
            		If a 'previous' profile already existed, it will be replaced. If the

            
            		<code class="code">--append</code> option is passed, old profile data in 'current' is left in place and

            
            		new profile data will be added to it, and the 'previous' profile (if one existed)

            
            		will remain untouched. To access the 'previous' profile, simply add a session

            
            		specification to the normal invocation of oprofile post-processing tools; for example:

            
            		</p>

            
                            <p>

            
            		</p>

            
                            <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                              <tr>

            
                                <td>

            
                                  <pre class="screen">opreport session:previous</pre>

            
                                </td>

            
                              </tr>

            
                            </table>

            
                            <p>

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--events / -e  [event1[,event2[,...]]]</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This option is for passing a comma-separated list of event specifications

            
            		for profiling. Each event spec is of the form:

            
            		</p>

            
                            <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                              <tr>

            
                                <td>

            
                                  <pre class="screen">name:count[:unitmask[:kernel[:user]]]</pre>

            
                                </td>

            
                              </tr>

            
                            </table>

            
                            <p>

            
            		When no event specification is given, the default event for the running

            
            		processor type will be used for profiling. Use <span class="command"><strong>ophelp</strong></span>

            
            		to list the available events for your processor type.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--separate-thread / -t</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This option categorizes samples by thread group ID (tgid) and thread ID (tid).

            
            		The <code class="code">--separate-thread</code> option is useful for seeing per-thread samples in

            
            		multi-threaded applications.  When used in conjuction with the <code class="code">--system-wide</code>

            
            		option, <code class="code">--separate-thread</code> is also useful for seeing per-process

            
            		(i.e., per-thread group) samples for the case where multiple processes are

            
            		executing the same program during a profiling run.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--separate-cpu / -c</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This option categorizes samples by cpu.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--session-dir / -d [path]</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This option specifies the session directory to hold the sample data. If not specified,

            
            		the data is saved in the <code class="filename">oprofile_data</code> directory on the current path.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">---lazy-conversion / -l</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Use this option to reduce the overhead of <span class="command"><strong>operf</strong></span> during profiling.

            
            		Normally, profile data received from the kernel is converted to OProfile format

            
            		during profiling time. This is typically not an issue when profiling a single

            
            		application. But when using the <code class="code">--system-wide</code> option, this on-the-fly

            
            		conversion process can cause noticeable overhead, particularly on busy

            
            		multi-processor systems. The <code class="code">--lazy-conversion</code> option directs

            
            		<span class="command"><strong>operf</strong></span> to wait until profiling is completed to do the conversion

            
            		of profile data.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--verbose / -V [level]</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		A comma-separated list of debugging control values used to increase the verbosity of the

            
            		output. Valid values are: debug, record, convert, misc, sfile, arcs, and the special value, 'all'.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--version -v </code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Show <span class="command"><strong>operf</strong></span> version.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--help / -h</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Show a help message.

            
            		</p>

            
                          </dd>

            
                        </dl>

            
                      </div>

            
                    </div>

            
                    <div class="sect1" title="2. Using opcontrol">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="controlling-daemon"></a>2. Using <span class="command"><strong>opcontrol</strong></span></h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            In this section we describe the configuration and control of the profiling system

            
            with opcontrol in more depth. See <a class="xref" href="#controlling-operf" title="1. Using operf">Section 1, &#8220;Using <span class="command"><strong>operf</strong></span>&#8221;</a> for a description

            
            of the preferred profiling method.

            
            </p>

            
                      <p>

            
            The <span class="command"><strong>opcontrol</strong></span> script has a default setup, but you

            
            can alter this with the options given below. In particular, you can select

            
            specific hardware events on which to base your profile. See <a class="xref" href="#controlling-operf" title="1. Using operf">Section 1, &#8220;Using <span class="command"><strong>operf</strong></span>&#8221;</a> for an

            
            introduction to hardware events and performance counter configuration.

            
            The event types and unit masks for your CPU are listed by <span class="command"><strong>opcontrol

            
            --list-events</strong></span> or <span class="command"><strong>ophelp</strong></span>.

            
            </p>

            
                      <p>

            
            The <span class="command"><strong>opcontrol</strong></span> script provides the following actions :

            
            </p>

            
                      <div class="variablelist">

            
                        <dl>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--init</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Loads the OProfile module if required and makes the OProfile driver

            
            		interface available.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--setup</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		    Followed by list arguments for profiling set up. List of arguments

            
            		    saved in <code class="filename">/root/.oprofile/daemonrc</code>.

            
            		    Giving this option is not necessary; you can just directly pass one

            
            		    of the setup options, e.g. <span class="command"><strong>opcontrol --no-vmlinux</strong></span>.

            
            		  </p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--status</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Show configuration information.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--start-daemon</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		    Start the oprofile daemon without starting actual profiling. The profiling

            
            		can then be started using <code class="option">--start</code>. This is useful for avoiding

            
            		measuring the cost of daemon startup, as <code class="option">--start</code> is a simple

            
            		write to a file in oprofilefs.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--start</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		    Start data collection with either arguments provided by <code class="option">--setup</code>

            
            		or information saved in <code class="filename">/root/.oprofile/daemonrc</code>. Specifying

            
            		the addition <code class="option">--verbose</code> makes the daemon generate lots of debug data

            
            		whilst it is running.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--dump</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		    Force a flush of the collected profiling data to the daemon.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--stop</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		    Stop data collection.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--shutdown</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		    Stop data collection and kill the daemon.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--reset</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		    Clears out data from current session, but leaves saved sessions.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term"><code class="option">--save=</code>session_name</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		    Save data from current session to session_name.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--deinit</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
                            Shuts down daemon. Unload the OProfile module and oprofilefs.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--list-events</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		    List event types and unit masks.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--help</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		    Generate usage messages.

            
            		</p>

            
                          </dd>

            
                        </dl>

            
                      </div>

            
                      <p>

            
            There are a number of possible settings, of which, only

            
            <code class="option">--vmlinux</code> (or <code class="option">--no-vmlinux</code>)

            
            is required. These settings are stored in <code class="filename">~/.oprofile/daemonrc</code>.

            
            </p>

            
                      <div class="variablelist">

            
                        <dl>

            
                          <dt>

            
                            <span class="term"><code class="option">--buffer-size=</code>num</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Number of samples in kernel buffer.

            
            		Buffer watershed needs to be tweaked when changing this value.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term"><code class="option">--buffer-watershed=</code>num</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Set kernel buffer watershed to num samples. When remain only

            
            		buffer-size - buffer-watershed free entries remain in the kernel buffer, data will be

            
            		flushed to the daemon.  Most useful values are in the range [0.25 - 0.5] * buffer-size.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term"><code class="option">--cpu-buffer-size=</code>num</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Number of samples in kernel per-cpu buffer. If you

            
            		profile at high rate, it can help to increase this if the log

            
            		file show excessive count of samples lost due to cpu buffer overflow.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term"><code class="option">--event=</code>[eventspec]</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Use the given performance counter event to profile.

            
            		See <a class="xref" href="#eventspec" title="4. Specifying performance counter events">Section 4, &#8220;Specifying performance counter events&#8221;</a> below.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term"><code class="option">--session-dir=</code>dir_path</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		    Create/use sample database out of directory <code class="filename">dir_path</code> instead of

            
            		the default location (/var/lib/oprofile).

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term"><code class="option">--separate=</code>[none,lib,kernel,thread,cpu,all]</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		By default, every profile is stored in a single file. Thus, for example,

            
            		samples in the C library are all accredited to the <code class="filename">/lib/libc.o</code>

            
            		profile. However, you choose to create separate sample files by specifying

            
            		one of the below options.

            
            		</p>

            
                            <div class="informaltable">

            
                              <table border="1">

            
                                <colgroup>

            
                                  <col />

            
                                  <col />

            
                                </colgroup>

            
                                <tbody>

            
                                  <tr>

            
                                    <td>

            
                                      <code class="option">none</code>

            
                                    </td>

            
                                    <td>No profile separation (default)</td>

            
                                  </tr>

            
                                  <tr>

            
                                    <td>

            
                                      <code class="option">lib</code>

            
                                    </td>

            
                                    <td>Create per-application profiles for libraries</td>

            
                                  </tr>

            
                                  <tr>

            
                                    <td>

            
                                      <code class="option">kernel</code>

            
                                    </td>

            
                                    <td>Create per-application profiles for the kernel and kernel modules</td>

            
                                  </tr>

            
                                  <tr>

            
                                    <td>

            
                                      <code class="option">thread</code>

            
                                    </td>

            
                                    <td>Create profiles for each thread and each task</td>

            
                                  </tr>

            
                                  <tr>

            
                                    <td>

            
                                      <code class="option">cpu</code>

            
                                    </td>

            
                                    <td>Create profiles for each CPU</td>

            
                                  </tr>

            
                                  <tr>

            
                                    <td>

            
                                      <code class="option">all</code>

            
                                    </td>

            
                                    <td>All of the above options</td>

            
                                  </tr>

            
                                </tbody>

            
                              </table>

            
                            </div>

            
                            <p>

            
            		Note  that <code class="option">--separate=kernel</code> also turns on <code class="option">--separate=lib</code>.


            		When using <code class="option">--separate=kernel</code>, samples in hardware interrupts, soft-irqs, or other

            
            		asynchronous kernel contexts are credited to the task currently running. This means you will see

            
            		seemingly nonsense profiles such as <code class="filename">/bin/bash</code> showing samples for the PPP modules,

            
            		etc.

            
            		</p>

            
                            <p>

            
            		Using <code class="option">--separate=thread</code> creates a lot

            
            		of sample files if you leave OProfile running for a while; it's most

            
            		useful when used for short sessions, or when using image filtering.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term"><code class="option">--callgraph=</code>#depth</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Enable call-graph sample collection with a maximum depth. Use 0 to disable

            
            		callgraph profiling.  NOTE: Callgraph support is available on a limited

            
            		number of platforms at this time; for example:

            
            		</p>

            
                            <p>

            
            		</p>

            
                            <div class="itemizedlist">

            
                              <ul class="itemizedlist" type="disc">

            
                                <li class="listitem">

            
                                  <p>x86 with 2.6 or higher kernel</p>

            
                                </li>

            
                                <li class="listitem">

            
                                  <p>ARM with 2.6 or higher kernel</p>

            
                                </li>

            
                                <li class="listitem">

            
                                  <p>PowerPC with 2.6.17 or higher kernel</p>

            
                                </li>

            
                              </ul>

            
                            </div>

            
                            <p>

            
            		</p>

            
                            <p>

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term"><code class="option">--image=</code>image,[images]|"all"</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Image filtering. If you specify one or more absolute

            
            		paths to binaries, OProfile will only produce profile results for those

            
            		binary images. This is useful for restricting the sometimes voluminous

            
            		output you may get otherwise, especially with

            
            		<code class="option">--separate=thread</code>. Note that if you are using

            
            		<code class="option">--separate=lib</code> or

            
            		<code class="option">--separate=kernel</code>, then if you specification an

            
            		application binary, the shared libraries and kernel code

            
            		<span class="emphasis"><em>are</em></span> included. Specify the value

            
            		"all" to profile everything (the default).

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term"><code class="option">--vmlinux=</code>file</span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		vmlinux kernel image.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--no-vmlinux</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Use this when you don't have a kernel vmlinux file, and you don't want

            
            		to profile the kernel. This still counts the total number of kernel samples,

            
            		but can't give symbol-based results for the kernel or any modules.

            
            		</p>

            
                          </dd>

            
                        </dl>

            
                      </div>

            
                      <div class="sect2" title="2.1. Examples">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="opcontrolexamples"></a>2.1. Examples</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <div class="sect3" title="2.1.1. Intel performance counter setup">

            
                          <div class="titlepage">

            
                            <div>

            
                              <div>

            
                                <h4 class="title"><a id="examplesperfctr"></a>2.1.1. Intel performance counter setup</h4>

            
                              </div>

            
                            </div>

            
                          </div>

            
                          <p>

            
            Here, we have a Pentium III running at 800MHz, and we want to look at where data memory

            
            references are happening most, and also get results for CPU time.

            
            </p>

            
                          <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                            <tr>

            
                              <td>

            
                                <pre class="screen">

            
            # opcontrol --event=CPU_CLK_UNHALTED:400000 --event=DATA_MEM_REFS:10000

            
            # opcontrol --vmlinux=/boot/2.6.0/vmlinux

            
            # opcontrol --start

            
            </pre>

            
                              </td>

            
                            </tr>

            
                          </table>

            
                        </div>

            
                        <div class="sect3" title="2.1.2. Starting the daemon separately">

            
                          <div class="titlepage">

            
                            <div>

            
                              <div>

            
                                <h4 class="title"><a id="examplesstartdaemon"></a>2.1.2. Starting the daemon separately</h4>

            
                              </div>

            
                            </div>

            
                          </div>

            
                          <p>

            
            Use <code class="option">--start-daemon</code> to avoid

            
            the profiler startup affecting results.

            
            </p>

            
                          <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                            <tr>

            
                              <td>

            
                                <pre class="screen">

            
            # opcontrol --vmlinux=/boot/2.6.0/vmlinux

            
            # opcontrol --start-daemon

            
            # my_favourite_benchmark --init

            
            # opcontrol --start ; my_favourite_benchmark --run ; opcontrol --stop

            
            </pre>

            
                              </td>

            
                            </tr>

            
                          </table>

            
                        </div>

            
                        <div class="sect3" title="2.1.3. Separate profiles for libraries and the kernel">

            
                          <div class="titlepage">

            
                            <div>

            
                              <div>

            
                                <h4 class="title"><a id="exampleseparate"></a>2.1.3. Separate profiles for libraries and the kernel</h4>

            
                              </div>

            
                            </div>

            
                          </div>

            
                          <p>

            
            Here, we want to see a profile of the OProfile daemon itself, including when

            
            it was running inside the kernel driver, and its use of shared libraries.

            
            </p>

            
                          <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                            <tr>

            
                              <td>

            
                                <pre class="screen">

            
            # opcontrol --separate=kernel --vmlinux=/boot/2.6.0/vmlinux

            
            # opcontrol --start

            
            # my_favourite_stress_test --run

            
            # opreport -l -p /lib/modules/2.6.0/kernel /usr/local/bin/oprofiled

            
            </pre>

            
                              </td>

            
                            </tr>

            
                          </table>

            
                        </div>

            
                        <div class="sect3" title="2.1.4. Profiling sessions">

            
                          <div class="titlepage">

            
                            <div>

            
                              <div>

            
                                <h4 class="title"><a id="examplessessions"></a>2.1.4. Profiling sessions</h4>

            
                              </div>

            
                            </div>

            
                          </div>

            
                          <p>

            
            It can often be useful to split up profiling data into several different

            
            time periods. For example, you may want to collect data on an application's

            
            startup separately from the normal runtime data. You can use the simple

            
            command <span class="command"><strong>opcontrol --save</strong></span> to do this. For example :

            
            </p>

            
                          <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                            <tr>

            
                              <td>

            
                                <pre class="screen">

            
            # opcontrol --save=blah

            
            </pre>

            
                              </td>

            
                            </tr>

            
                          </table>

            
                          <p>

            
            will create a sub-directory in <code class="filename">$SESSION_DIR/samples</code> containing the samples

            
            up to that point (the current session's sample files are moved into this

            
            directory). You can then pass this session name as a parameter to the post-profiling

            
            analysis tools, to only get data up to the point you named the

            
            session. If you do not want to save a session, you can do

            
            <span class="command"><strong>rm -rf $SESSION_DIR/samples/sessionname</strong></span> or, for the

            
            current session, <span class="command"><strong>opcontrol --reset</strong></span>.

            
            </p>

            
                        </div>

            
                      </div>

            
                    </div>

            
                    <div class="sect1" title="3. Setting up the JIT profiling feature">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="setup-jit"></a>3. Setting up the JIT profiling feature</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            		To gather information about JITed code from a virtual machine,

            
            		it needs to be instrumented with an agent library. We use the

            
            		agent libraries for Java in the following example. To use the

            
            		Java profiling feature, you must build OProfile with the "--with-java" option

            
                            (<a class="xref" href="#install" title="7. Installation">Section 7, &#8220;Installation&#8221;</a>).


            	</p>

            
                      <div class="sect2" title="3.1. JVM instrumentation">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="setup-jit-jvm"></a>3.1. JVM instrumentation</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            			Add this to the startup parameters of the JVM (for JVMTI):


            			</p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen"><code xmlns="http://www.w3.org/1999/xhtml" class="option">-agentpath:&lt;libdir&gt;/libjvmti_oprofile.so[=&lt;options&gt;]</code> </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            			or

            
            			</p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen"><code xmlns="http://www.w3.org/1999/xhtml" class="option">-agentlib:jvmti_oprofile[=&lt;options&gt;]</code> </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            		</p>

            
                        <p>

            
            			The JVMPI agent implementation is enabled with the command line option

            
            			</p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen"><code xmlns="http://www.w3.org/1999/xhtml" class="option">-Xrunjvmpi_oprofile[:&lt;options&gt;]</code> </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            		</p>

            
                        <p>

            
            			Currently, there is just one option available -- <code class="option">debug</code>. For JVMPI,

            
            			the convention for specifying an option is <code class="option">option_name=[yes|no]</code>.

            
            			For JVMTI, the option specification is simply the option name, implying

            
            			"yes"; no option specified implies "no".

            
            		</p>

            
                        <p>

            
                                    The agent library (installed in <code class="filename">&lt;oprof_install_dir&gt;/lib/oprofile</code>)

            
                                    needs to be in the library search path (e.g. add the library directory

            
                                    to <code class="constant">LD_LIBRARY_PATH</code>). If the command line of

            
                                    the JVM is not accessible, it may be buried within shell scripts or a

            
                                    launcher program. It may also be possible to set an environment variable to add

            
                                    the instrumentation.

            
                                    For Sun JVMs this is <code class="constant">JAVA_TOOL_OPTIONS</code>. Please check

            
                                    your JVM documentation for

            
                                    further information on the agent startup options.

            
                            </p>

            
                      </div>

            
                    </div>

            
                    <div class="sect1" title="4. Using oprof_start">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="oprofile-gui"></a>4. Using <span class="command"><strong>oprof_start</strong></span></h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            The <span class="command"><strong>oprof_start</strong></span> application provides a convenient way to start the profiler.

            
            Note that <span class="command"><strong>oprof_start</strong></span> is just a wrapper around the <span class="command"><strong>opcontrol</strong></span> script,

            
            so it does not provide more services than the script itself.

            
            </p>

            
                      <p>

            
            After <span class="command"><strong>oprof_start</strong></span> is started you can select the event type for each counter;

            
            the sampling rate and other related parameters are explained in <a class="xref" href="#controlling-daemon" title="2. Using opcontrol">Section 2, &#8220;Using <span class="command"><strong>opcontrol</strong></span>&#8221;</a>.

            
            The "Configuration" section allows you to set general parameters such as the buffer size, kernel filename

            
            etc. The counter setup interface should be self-explanatory; <a class="xref" href="#hardware-counters" title="5.1. Hardware performance counters">Section 5.1, &#8220;Hardware performance counters&#8221;</a> and related

            
            links contain information on using unit masks.

            
            </p>

            
                      <p>

            
            A status line shows the current status of the profiler: how long it has been running, and the average

            
            number of interrupts received per second and the total, over all processors.

            
            Note that quitting <span class="command"><strong>oprof_start</strong></span> does not stop the profiler.

            
            </p>

            
                      <p>

            
            Your configuration is saved in the same file as <span class="command"><strong>opcontrol</strong></span> uses; that is,

            
            <code class="filename">~/.oprofile/daemonrc</code>.

            
            </p>

            
                      <p>

            
            </p>

            
                      <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><h3 class="title">Note</h3><span class="command"><strong>oprof_start</strong></span> does not currently support <span class="command"><strong>operf</strong></span>.</div>

            
                      <p>

            
            </p>

            
                    </div>

            
                    <div class="sect1" title="5. Configuration details">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="detailed-parameters"></a>5. Configuration details</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <div class="sect2" title="5.1. Hardware performance counters">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="hardware-counters"></a>5.1. Hardware performance counters</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>Most processor models include performance monitor units that can be configured to monitor (count)

            
            various types of hardware events.  This section is where you can find architecture-specific information

            
            to help you use these events for profiling. You do not really need to read this section unless you are interested in using

            
            events other than the default event chosen by OProfile.

            
            </p>

            
                        <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;">

            
                          <h3 class="title">Note</h3>

            
                          <p>

            
            Your CPU type may not include the requisite support for hardware performance counters, in which case

            
            you must use OProfile in timer mode (see <a class="xref" href="#timer" title="5.2. OProfile in timer interrupt mode">Section 5.2, &#8220;OProfile in timer interrupt mode&#8221;</a>).

            
            </p>

            
                        </div>

            
                        <p>

            
            The Intel hardware performance counters are detailed in the Intel IA-32 Architecture Manual, Volume 3, available

            
            from <a class="ulink" href="http://developer.intel.com/">http://developer.intel.com/</a>.

            
            The AMD Athlon/Opteron/Phenom/Turion implementation is detailed in <a class="ulink" href="http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/22007.pdf">

            
            http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/22007.pdf</a>.

            
            For IBM PowerPC processors, documentation is available at <a class="ulink" href="https://www.power.org/">

            
            https://www.power.org/</a>. For example, <a class="ulink" href="https://www.power.org/events/Power7">

            
            https://www.power.org/events/Power7</a> contains specific information on the performance

            
            monitor unit for the IBM POWER7.

            
            </p>

            
                        <p>

            
            These processors are capable of delivering an interrupt when a counter overflows.

            
            This is the basic mechanism on which OProfile is based. The delivery mode is <acronym class="acronym">NMI</acronym>,

            
            so blocking interrupts in the kernel does not prevent profiling. When the interrupt handler is called,

            
            the current <acronym class="acronym">PC</acronym> value and the current task are recorded into the profiling structure.

            
            This allows the overflow event to be attached to a specific assembly instruction in a binary image.

            
            OProfile receives this data from the kernel and writes it to the sample files.

            
            </p>

            
                        <p>

            
            If we use an event such as <code class="constant">CPU_CLK_UNHALTED</code> or <code class="constant">INST_RETIRED</code>

            
            (<code class="constant">GLOBAL_POWER_EVENTS</code> or <code class="constant">INSTR_RETIRED</code>, respectively, on the Pentium 4), we can

            
            use the overflow counts as an estimate of actual time spent in each part of code. Alternatively we can profile interesting

            
            data such as the cache behaviour of routines with the other available counters.

            
            </p>

            
                        <p>

            
            However there are several caveats. First, there are those issues listed in the Intel manual. There is a delay

            
            between the counter overflow and the interrupt delivery that can skew results on a small scale - this means

            
            you cannot rely on the profiles at the instruction level as being perfectly accurate.

            
            If you are using an "event-mode" counter such as the cache counters, a count registered against it doesn't mean

            
            that it is responsible for that event. However, it implies that the counter overflowed in the dynamic

            
            vicinity of that instruction, to within a few instructions. Further details on this problem can be found in

            
            <a class="xref" href="#interpreting" title="Chapter 5. Interpreting profiling results">Chapter 5, <i>Interpreting profiling results</i></a> and also in the Digital paper "ProfileMe: A Hardware Performance Counter".

            
            </p>

            
                        <p>

            
            Each counter has several configuration parameters.

            
            First, there is the unit mask: this simply further specifies what to count.

            
            Second, there is the counter value, discussed below. Third, there is a parameter whether to increment counts

            
            whilst in kernel or user space. You can configure these separately for each counter.

            
            </p>

            
                        <p>

            
            After each overflow event, the counter will be re-initialized

            
            such that another overflow will occur after this many events have been counted. Thus, higher

            
            values mean less-detailed profiling, and lower values mean more detail, but higher overhead.

            
            Picking a good value for this

            
            parameter is, unfortunately, somewhat of a black art. It is of course dependent on the event

            
            you have chosen.

            
            Specifying too large a value will mean not enough interrupts are generated

            
            to give a realistic profile (though this problem can be ameliorated by profiling for <span class="emphasis"><em>longer</em></span>).

            
            Specifying too small a value can lead to higher performance overhead.

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="5.2. OProfile in timer interrupt mode">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="timer"></a>5.2. OProfile in timer interrupt mode</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            Some CPU types do not provide the needed hardware support to use the hardware performance counters. This includes

            
            some laptops, classic Pentiums, and other CPU types not yet supported by OProfile (such as Cyrix).

            
            On these machines, OProfile falls back to using the timer interrupt for profiling,

            
            back to using the real-time clock interrupt to collect samples.  In timer mode, OProfile

            
            is not able to profile code that has interrupts disabled.

            
            </p>

            
                        <p>

            
            You can force use of the timer interrupt by using the <code class="option">timer=1</code> module

            
            parameter (or <code class="option">oprofile.timer=1</code> on the boot command line if OProfile is

            
            built-in).  If OProfile was built as a kernel module, then you must pass the 'timer=1'

            
            parameter with the modprobe command.  Do this before executing 'opcontrol --init' or

            
            edit the opcontrol command's invocation of modprobe to pass the 'timer=1' parameter.


            </p>

            
                        <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><h3 class="title">Note</h3>Timer mode is only available using the legacy <span class="command"><strong>opcontrol</strong></span> command.</div>

            
                        <p>

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="5.3. Pentium 4 support">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="p4"></a>5.3. Pentium 4 support</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            The Pentium 4 / Xeon performance counters are organized around 3 types of model specific registers (MSRs): 45 event

            
            selection control registers (ESCRs), 18 counter configuration control registers (CCCRs) and 18 counters. ESCRs describe a

            
            particular set of events which are to be recorded, and CCCRs bind ESCRs to counters and configure their

            
            operation. Unfortunately the relationship between these registers is quite complex; they cannot all be used with one

            
            another at any time. There is, however, a subset of 8 counters, 8 ESCRs, and 8 CCCRs which can be used independently of

            
            one another, so OProfile only accesses those registers, treating them as a bank of 8 "normal" counters, similar

            
            to those in the P6 or Athlon/Opteron/Phenom/Turion families of CPU.

            
            </p>

            
                        <p>

            
            There is currently no support for Precision Event-Based Sampling (PEBS), nor any advanced uses of the Debug Store

            
            (DS). Current support is limited to the conservative extension of OProfile's existing interrupt-based model described

            
            above.

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="5.4. Intel Itanium 2 support">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="ia64"></a>5.4. Intel Itanium 2 support</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            The Itanium 2 performance monitoring unit (PMU) organizes the counters as four

            
            pairs of performance event monitoring registers. Each pair is composed of a

            
            Performance Monitoring Configuration (PMC) register and Performance Monitoring

            
            Data (PMD) register.  The PMC selects the performance event being monitored and

            
            the PMD determines the sampling interval. The IA64 Performance Monitoring Unit

            
            (PMU) triggers sampling with maskable interrupts. Thus, samples will not occur

            
            in sections of the IA64 kernel where interrupts are disabled.

            
            </p>

            
                        <p>

            
            None of the advance features of the Itanium 2 performance monitoring unit

            
            such as opcode matching, address range matching, or precise event sampling are

            
            supported by this version of OProfile.  The Itanium 2 support only maps OProfile's

            
            existing interrupt-based model to the PMU hardware.

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="5.5. PowerPC64 support">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="ppc64"></a>5.5. PowerPC64 support</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            The performance monitoring unit (PMU) for the IBM PowerPC 64-bit processors

            
            consists of between 4 and 8 counters (depending on the model), plus three

            
            special purpose registers used for programming the counters -- MMCR0, MMCR1,

            
            and MMCRA.  Advanced features such as instruction matching and thresholding are

            
            not supported by this version of OProfile.

            
            </p>

            
                        <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><h3 class="title">Note</h3>Later versions of the IBM POWER5+ processor (beginning with revision 3.0)

            
            run the performance monitor unit in POWER6 mode, effectively removing OProfile's

            
            access to counters 5 and 6.  These two counters are dedicated to counting

            
            instructions completed and cycles, respectively.  In POWER6 mode, however, the

            
            counters do not generate an interrupt on overflow and so are unusable by

            
            OProfile.  Kernel versions 2.6.23 and higher will recognize this mode

            
            and export "ppc64/power5++" as the cpu_type to the oprofilefs pseudo filesystem.

            
            OProfile userspace responds to this cpu_type by removing these counters from

            
            the list of potential events to count.  Without this kernel support, attempts

            
            to profile using an event from one of these counters will yield incorrect

            
            results -- typically, zero (or near zero) samples in the generated report.

            
            </div>

            
                        <p>

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="5.6. Cell Broadband Engine support">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="cell-be"></a>5.6. Cell Broadband Engine support</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            The Cell Broadband Engine (CBE) processor core consists of a PowerPC Processing

            
            Element (PPE) and 8 Synergistic Processing Elements (SPE).  PPEs and SPEs each

            
            consist of a processing unit (PPU and SPU, respectively) and other hardware

            
            components, such as memory controllers.

            
            </p>

            
                        <p>

            
            A PPU has two hardware threads (aka "virtual CPUs").  The performance monitor

            
            unit of the CBE collects event information on one hardware thread at a time.

            
            Therefore, when profiling PPE events,

            
            OProfile collects the profile based on the selected events by time slicing the

            
            performance counter hardware between the two threads.   The user must ensure the

            
            collection interval is long enough so that the time spent collecting data for

            
            each PPU is sufficient to obtain a good profile.

            
            </p>

            
                        <p>

            
            To profile an SPU application, the user should specify the SPU_CYCLES event.

            
            When starting OProfile with SPU_CYCLES, the opcontrol script enforces certain

            
            separation parameters (separate=cpu,lib) to ensure that sufficient information

            
            is collected in the sample data in order to generate a complete report.  The

            
            --merge=cpu option can be used to obtain a more readable report if analyzing

            
            the performance of each separate SPU is not necessary.

            
            </p>

            
                        <p>

            
            Profiling with an SPU event (events 4100 through 4163) is not compatible with any other

            
            event.  Further more, only one SPU event can be specified at a time.  The hardware only

            
            supports profiling on one SPU per node at a time.  The OProfile kernel code time slices

            
            between the eight SPUs to collect data on all SPUs.

            
            </p>

            
                        <p>

            
            SPU profile reports have some unique characteristics compared to reports for

            
            standard architectures:

            
            </p>

            
                        <div class="itemizedlist">

            
                          <ul class="itemizedlist" type="disc">

            
                            <li class="listitem">Typically no "app name" column.  This is really standard OProfile behavior

            
            when the report contains samples for just a single application, which is

            
            commonly the case when profiling SPUs.</li>

            
                            <li class="listitem">"CPU" equates to "SPU"</li>

            
                            <li class="listitem">Specifying '--long-filenames' on the opreport command does not always result

            
            in long filenames.  This happens when the SPU application code is embedded in

            
            the PPE executable or shared library.  The embedded SPU ELF data contains only the

            
            short filename (i.e., no path information) for the SPU binary file that was used as

            
            the source for embedding.   The reason that just the short filename is used is because

            
            the original SPU binary file may not exist or be accessible at runtime.  The performance

            
            analyst must have sufficient knowledge of the application to be able to correlate the

            
            SPU binary image names found in the  report to the application's source files.

            
            <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><h3 class="title">Note</h3>

            
            Compile the application with -g and generate the OProfile report

            
            with -g to facilitate finding the right source file(s) on which to focus.

            
            </div></li>

            
                          </ul>

            
                        </div>

            
                      </div>

            
                      <div class="sect2" title="5.7. AMD64 (x86_64) Instruction-Based Sampling (IBS) support">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="amd-ibs-support"></a>5.7. AMD64 (x86_64) Instruction-Based Sampling (IBS) support</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            Instruction-Based Sampling (IBS) is a new performance measurement technique

            
            available on AMD Family 10h processors. Traditional performance counter

            
            sampling is not precise enough to isolate performance issues to individual

            
            instructions. IBS, however, precisely identifies instructions which are not

            
            making the best use of the processor pipeline and memory hierarchy.

            
            For more information, please refer to the "Instruction-Based Sampling:

            
            A New Performance Analysis Technique for AMD Family 10h Processors" (

            
            <a class="ulink" href="http://developer.amd.com/assets/AMD_IBS_paper_EN.pdf">

            
            http://developer.amd.com/assets/AMD_IBS_paper_EN.pdf</a>).

            
            There are two types of IBS profile types, described in the following sections.

            
            </p>

            
                        <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><h3 class="title">Note</h3>Profiling on IBS events is only supported with legacy mode profiling

            
            (i.e., with <span class="command"><strong>opcontrol</strong></span>).</div>

            
                        <p>

            
            </p>

            
                        <div class="sect3" title="5.7.1. IBS Fetch">

            
                          <div class="titlepage">

            
                            <div>

            
                              <div>

            
                                <h4 class="title"><a id="ibs-fetch"></a>5.7.1. IBS Fetch</h4>

            
                              </div>

            
                            </div>

            
                          </div>

            
                          <p>

            
            IBS fetch sampling is a statistical sampling method which counts completed

            
            fetch operations. When the number of completed fetch operations reaches the

            
            maximum fetch count (the sampling period), IBS tags the fetch operation and

            
            monitors that operation until it either completes or aborts. When a tagged

            
            fetch completes or aborts, a sampling interrupt is generated and an IBS fetch

            
            sample is taken. An IBS fetch sample contains a timestamp, the identifier of

            
            the interrupted process, the virtual fetch address, and several event flags

            
            and values that describe what happened during the fetch operation.

            
            </p>

            
                        </div>

            
                        <div class="sect3" title="5.7.2. IBS Op">

            
                          <div class="titlepage">

            
                            <div>

            
                              <div>

            
                                <h4 class="title"><a id="ibs-op"></a>5.7.2. IBS Op</h4>

            
                              </div>

            
                            </div>

            
                          </div>

            
                          <p>

            
            IBS op sampling selects, tags, and monitors macro-ops as issued from AMD64

            
            instructions. Two options are available for selecting ops for sampling:

            
            </p>

            
                          <div class="itemizedlist">

            
                            <ul class="itemizedlist" type="disc">

            
                              <li class="listitem">

            
            Cycles-based selection counts CPU clock cycles. The op is tagged and monitored

            
            when the count reaches a threshold (the sampling period) and a valid op is

            
            available.

            
            </li>

            
                              <li class="listitem">

            
            Dispatched op-based selection counts dispatched macro-ops.

            
            When the count reaches a threshold, the next valid op is tagged and monitored.

            
            </li>

            
                            </ul>

            
                          </div>

            
                          <p>

            
            In both cases, an IBS sample is generated only if the tagged op retires.

            
            Thus, IBS op event information does not measure speculative execution activity.

            
            The execution stages of the pipeline monitor the tagged macro-op. When the

            
            tagged macro-op retires, a sampling interrupt is generated and an IBS op

            
            sample is taken. An IBS op sample contains a timestamp, the identifier of

            
            the interrupted process, the virtual address of the AMD64 instruction from

            
            which the op was issued, and several event flags and values that describe

            
            what happened when the macro-op executed.

            
            </p>

            
                        </div>

            
                        <p>

            
            Enabling IBS profiling is done simply by specifying IBS performance events

            
            through the "--event=" options. These events are listed in the

            
            <code class="function">opcontrol --list-events</code>.

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
            opcontrol --event=IBS_FETCH_XXX:&lt;count&gt;:&lt;um&gt;:&lt;kernel&gt;:&lt;user&gt;

            
            opcontrol --event=IBS_OP_XXX:&lt;count&gt;:&lt;um&gt;:&lt;kernel&gt;:&lt;user&gt;


            Note: * All IBS fetch event must have the same event count and unitmask,

            
                    as do those for IBS op.

            
            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                      </div>

            
                      <div class="sect2" title="5.8. IBM System z hardware sampling support">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="systemz"></a>5.8. IBM System z hardware sampling support</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            IBM System z provides a facility which does instruction sampling as

            
            part of the CPU.  This has great advantages over the timer based

            
            sampling approach like better sampling resolution with less overhead

            
            and the possibility to get samples within code sections where

            
            interrupts are disabled (useful especially for Linux kernel code).

            
            </p>

            
                        <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><h3 class="title">Note</h3>Profiling with the instruction sampling facility is currently only supported

            
            with legacy mode profiling (i.e., with <span class="command"><strong>opcontrol</strong></span>).</div>

            
                        <p>

            
            A public description of the System z CPU-Measurement Facilities can be

            
            found here:

            
            <a class="ulink" href="http://www-01.ibm.com/support/docview.wss?uid=isg26fcd1cc32246f4c8852574ce0044734a">The Load-Program-Parameter and CPU-Measurement Facilities</a>

            
            </p>

            
                        <p>

            
            System z hardware sampling can be used for Linux instances in LPAR

            
            mode. The hardware sampling support used by OProfile was introduced

            
            for System z10 in October 2008.

            
            </p>

            
                        <p>

            
            To enable hardware sampling for an LPAR you must activate the LPAR

            
            with authorization for basic sampling control. See the "Support

            
            Element Operations Guide" for your mainframe system for more

            
            information.

            
            </p>

            
                        <p>

            
            The hardware sampling facility can be enabled and disabled using the

            
            event interface.  A `virtual' counter 0 has been defined that only supports

            
            a single event, HWSAMPLING. By default the HWSAMPLING event is

            
            enabled on machines providing the facility.  For both events only the

            
            `count', `kernel' and `user' options are evaluated by the kernel

            
            module.

            
            </p>

            
                        <p>

            
            The `count' value is the sampling rate as it is passed to the CPU

            
            measurement facility.  A sample will be taken by the hardware every

            
            `count' cycles. Using low values here will quickly fill up the

            
            sampling buffers and will generate CPU load on the OProfile daemon and

            
            the kernel module being busy flushing the hardware buffers.  This

            
            might considerably impact the workload to be profiled.

            
            </p>

            
                        <p>

            
            The unit mask `um' is required to be zero.

            
            </p>

            
                        <p>

            
            The opcontrol tool provides a new option specific to System z

            
            hardware sampling:

            
            </p>

            
                        <div class="itemizedlist">

            
                          <ul class="itemizedlist" type="disc">

            
                            <li class="listitem">--s390hwsampbufsize="num": Number of 2MB areas

            
            used per CPU for storing sample data.  The best

            
            size for the sample memory depends on the particular system and the

            
            workload to be measured.  Providing the sampler with too little memory

            
            results in lost samples. Reserving too much system memory for the

            
            sampler impacts the overall performance and, hence, also the workload

            
            to be measured.</li>

            
                          </ul>

            
                        </div>

            
                        <p>

            
            A special counter <code class="filename">/dev/oprofile/timer</code> is provided

            
            by the kernel module allowing to switch back to timer mode sampling

            
            dynamically.  The TIMER event is limited to be used only with this

            
            counter.  The TIMER event can be specified using the

            
            <code class="option">--event=</code> as with every other event.

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">opcontrol --event=TIMER:1</pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            On z10 or later machines the default event is set to TIMER in case the

            
            hardware sampling facility is not available.

            
            </p>

            
                        <p>

            
            Although required, the 'count' parameter of the TIMER event is

            
            ignored.  The value may eventually be used for timer based sampling

            
            with a configurable sampling frequency, but this is currently not

            
            supported.

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="5.9. Dangerous counter settings">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="misuse"></a>5.9. Dangerous counter settings</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            OProfile is a low-level profiler which allows continuous profiling with a low-overhead cost.

            
            When using OProfile legacy mode profiling, it may be possible to configure such a low a counter reset value

            
            (i.e., high sampling rate) that the system can become overloaded with counter interrupts and your

            
            system's responsiveness may be severely impacted. Whilst some validation is done on the <code class="code">count</code>

            
            values you pass to <span class="command"><strong>opcontrol</strong></span> with your event specification, it is not foolproof.

            
            </p>

            
                        <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;">

            
                          <h3 class="title">Note</h3>

            
                          <p>

            
            This can happen as follows: When the profiler count

            
            reaches zero, an NMI handler is called which stores the sample values in an internal buffer, then resets the counter

            
            to its original value. If the reset count you specified is very low, a pending NMI can be sent before the NMI handler has

            
            completed. Due to the priority of the NMI, the pending interrupt is delivered immediately after

            
            completion of the previous interrupt handler, and control never returns to other parts of the system.

            
            If all processors are stuck in this mode, the system will appear to be frozen.

            
            </p>

            
                        </div>

            
                        <p>If this happens, it will be impossible to bring the system back to a workable state.

            
            There is no way to provide real security against this happening, other than making sure to use a reasonable value

            
            for the counter reset. For example, setting <code class="constant">CPU_CLK_UNHALTED</code> event type with a ridiculously low reset count (e.g. 500)

            
            is likely to freeze the system.

            
            </p>

            
                        <p>

            
            In short : <span class="command"><strong>Don't try a foolish sample count value</strong></span>. Unfortunately the definition of a foolish value

            
            is really dependent on the event type. If ever in doubt, post a message to </p>

            
                        <div class="address">

            
                          <p><code class="email">&lt;<a class="email" href="mailto:oprofile-list@lists.sf.net">oprofile-list@lists.sf.net</a>&gt;</code>.</p>

            
                        </div>

            
                        <p>

            
            </p>

            
                        <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><h3 class="title">Note</h3>

            
            The scenario described above cannot occur if you use <span class="command"><strong>operf</strong></span> for profiling instead of

            
            <span class="command"><strong>opcontrol</strong></span>, because the perf_events kernel subsystem automatically detects when performance monitor

            
            interrupts are arriving at a dangerous level and will throttle back the sampling rate.

            
            </div>

            
                      </div>

            
                    </div>

            
                  </div>

            
                  <div class="chapter" title="Chapter 4. Obtaining profiling results">

            
                    <div class="titlepage">

            
                      <div>

            
                        <div>

            
                          <h2 class="title"><a id="results"></a>Chapter 4. Obtaining profiling results</h2>

            
                        </div>

            
                      </div>

            
                    </div>

            
                    <div class="toc">

            
                      <p>

            
                        <b>Table of Contents</b>

            
                      </p>

            
                      <dl>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#profile-spec">1. Profile specifications</a>

            
                          </span>

            
                        </dt>

            
                        <dd>

            
                          <dl>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#profile-spec-examples">1.1. Examples</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#profile-spec-details">1.2. Profile specification parameters</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#locating-and-managing-binary-images">1.3. Locating and managing binary images</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#no-results">1.4. What to do when you don't get any results</a>

            
                              </span>

            
                            </dt>

            
                          </dl>

            
                        </dd>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#opreport">2. Image summaries and symbol summaries (<span class="command"><strong>opreport</strong></span>)</a>

            
                          </span>

            
                        </dt>

            
                        <dd>

            
                          <dl>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#opreport-merging">2.1. Merging separate profiles</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#opreport-comparison">2.2. Side-by-side multiple results</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#opreport-callgraph">2.3. Callgraph output</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#opreport-diff">2.4. Differential profiles with <span class="command"><strong>opreport</strong></span></a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#opreport-anon">2.5. Anonymous executable mappings</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#opreport-xml">2.6. XML formatted output</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#opreport-options">2.7. Options for <span class="command"><strong>opreport</strong></span></a>

            
                              </span>

            
                            </dt>

            
                          </dl>

            
                        </dd>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#opannotate">3. Outputting annotated source (<span class="command"><strong>opannotate</strong></span>)</a>

            
                          </span>

            
                        </dt>

            
                        <dd>

            
                          <dl>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#opannotate-finding-source">3.1. Locating source files</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#opannotate-details">3.2. Usage of <span class="command"><strong>opannotate</strong></span></a>

            
                              </span>

            
                            </dt>

            
                          </dl>

            
                        </dd>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#getting-jit-reports">4. OProfile results with JIT samples</a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#opgprof">5. <span class="command"><strong>gprof</strong></span>-compatible output (<span class="command"><strong>opgprof</strong></span>)</a>

            
                          </span>

            
                        </dt>

            
                        <dd>

            
                          <dl>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#opgprof-details">5.1. Usage of <span class="command"><strong>opgprof</strong></span></a>

            
                              </span>

            
                            </dt>

            
                          </dl>

            
                        </dd>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#oparchive">6. Analyzing profile data on another system (<span class="command"><strong>oparchive</strong></span>)</a>

            
                          </span>

            
                        </dt>

            
                        <dd>

            
                          <dl>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#oparchive-details">6.1. Usage of <span class="command"><strong>oparchive</strong></span></a>

            
                              </span>

            
                            </dt>

            
                          </dl>

            
                        </dd>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#opimport">7. Converting sample database files (<span class="command"><strong>opimport</strong></span>)</a>

            
                          </span>

            
                        </dt>

            
                        <dd>

            
                          <dl>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#opimport-details">7.1. Usage of <span class="command"><strong>opimport</strong></span></a>

            
                              </span>

            
                            </dt>

            
                          </dl>

            
                        </dd>

            
                      </dl>

            
                    </div>

            
                    <p>

            
            OK, so the profiler has been running, but it's not much use unless we can get some data out. Sometimes,

            
            OProfile does a little <span class="emphasis"><em>too</em></span> good a job of keeping overhead low, and no data reaches

            
            the profiler. This can happen on lightly-loaded machines. If you're using OPorifle legacy mode, you can

            
            force a dump at any time with :

            
            </p>

            
                    <p>

            
                      <span class="command">

            
                        <strong>opcontrol --dump</strong>

            
                      </span>

            
                    </p>

            
                    <p>This ensures that any profile data collected by the <span class="command"><strong>oprofiled</strong></span> daemon has been flusehd

            
            to disk.  Remember to do a <code class="code">dump</code>, <code class="code">stop</code>, <code class="code">shutdown</code>, or <code class="code">deinit</code>

            
            before complaining there is no profiling data!

            
            </p>

            
                    <p>

            
            Now that we've got some data, it has to be processed. That's the job of <span class="command"><strong>opreport</strong></span>,

            
            <span class="command"><strong>opannotate</strong></span>, or <span class="command"><strong>opgprof</strong></span>.

            
            </p>

            
                    <div class="sect1" title="1. Profile specifications">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="profile-spec"></a>1. Profile specifications</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            All of the analysis tools take a <span class="emphasis"><em>profile specification</em></span>.

            
            This is a set of definitions that describe which actual profiles should be

            
            examined. The simplest profile specification is empty: this will match all

            
            the available profile files for the current session (this is what happens

            
            when you do <span class="command"><strong>opreport</strong></span>).

            
            </p>

            
                      <p>

            
            Specification parameters are of the form <code class="option">name:value[,value]</code>.

            
            For example, if I wanted to get a combined symbol summary for

            
            <code class="filename">/bin/myprog</code> and <code class="filename">/bin/myprog2</code>,

            
            I could do <span class="command"><strong>opreport -l image:/bin/myprog,/bin/myprog2</strong></span>.

            
            As a special case, you don't actually need to specify the <code class="option">image:</code>

            
            part here: anything left on the command line is assumed to be an

            
            <code class="option">image:</code> name. Similarly, if no <code class="option">session:</code>

            
            is specified, then <code class="option">session:current</code> is assumed ("current"

            
            is a special name of the current / last profiling session).

            
            </p>

            
                      <p>

            
            In addition to the comma-separated list shown above, some of the

            
            specification parameters can take <span class="command"><strong>glob</strong></span>-style

            
            values. For example, if I want to see image summaries for all

            
            binaries profiled in <code class="filename">/usr/bin/</code>, I could do

            
            <span class="command"><strong>opreport image:/usr/bin/\*</strong></span>. Note the necessity

            
            to escape the special character from the shell.

            
            </p>

            
                      <p>

            
            For <span class="command"><strong>opreport</strong></span>, profile specifications can be used to

            
            define two profiles, giving differential output. This is done by

            
            enclosing each of the two specifications within curly braces, as shown

            
            in the examples below. Any specifications outside of curly braces are

            
            shared across both.

            
            </p>

            
                      <div class="sect2" title="1.1. Examples">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="profile-spec-examples"></a>1.1. Examples</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            Image summaries for all profiles with <code class="constant">DATA_MEM_REFS</code>

            
            samples in the saved session called "stresstest" :

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
            # opreport session:stresstest event:DATA_MEM_REFS

            
            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            Symbol summary for the application called "test_sym53c8xx,9xx". Note the

            
            escaping is necessary as <code class="option">image:</code> takes a comma-separated list.

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
            # opreport -l ./test/test_sym53c8xx\,9xx

            
            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            Image summaries for all binaries in the <code class="filename">test</code> directory,

            
            excepting <code class="filename">boring-test</code> :

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
            # opreport image:./test/\* image-exclude:./test/boring-test

            
            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            Differential profile of a binary stored in two archives :

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
            # opreport -l /bin/bash { archive:./orig } { archive:./new }

            
            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            Differential profile of an archived binary with the current session :

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
            # opreport -l /bin/bash { archive:./orig } { }

            
            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                      </div>

            
                      <div class="sect2" title="1.2. Profile specification parameters">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="profile-spec-details"></a>1.2. Profile specification parameters</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <div class="variablelist">

            
                          <dl>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">archive:</code>

            
                                <span class="emphasis">

            
                                  <em>archivepath</em>

            
                                </span>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            		A path to an archive made with <span class="command"><strong>oparchive</strong></span>.

            
            		Absence of this tag, unlike others, means "the current system",

            
            		equivalent to specifying "archive:".

            
            		</p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">session:</code>

            
                                <span class="emphasis">

            
                                  <em>sessionlist</em>

            
                                </span>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            		A comma-separated list of session names to resolve in. Absence of this

            
            		tag, unlike others, means "the current session", equivalent to

            
            		specifying "session:current".

            
            		</p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">session-exclude:</code>

            
                                <span class="emphasis">

            
                                  <em>sessionlist</em>

            
                                </span>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
                            A comma-separated list of sessions to exclude.

            
            		</p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">image:</code>

            
                                <span class="emphasis">

            
                                  <em>imagelist</em>

            
                                </span>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
                            A comma-separated list of image names to resolve. Each entry may be relative

            
                            path, <span class="command"><strong>glob</strong></span>-style name, or full path, e.g.</p>

            
                              <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                                <tr>

            
                                  <td>

            
                                    <pre class="screen">opreport 'image:/usr/bin/oprofiled,*op*,./opreport'</pre>

            
                                  </td>

            
                                </tr>

            
                              </table>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">image-exclude:</code>

            
                                <span class="emphasis">

            
                                  <em>imagelist</em>

            
                                </span>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            		Same as <code class="option">image:</code>, but the matching images are excluded.

            
            		</p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">lib-image:</code>

            
                                <span class="emphasis">

            
                                  <em>imagelist</em>

            
                                </span>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            		Same as <code class="option">image:</code>, but only for images that are for

            
            		a particular primary binary image (namely, an application). This only

            
            		makes sense to use if you're using <code class="option">--separate</code>.

            
            		This includes kernel modules and the kernel when using

            
            		<code class="option">--separate=kernel</code>.

            
            		</p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">lib-image-exclude:</code>

            
                                <span class="emphasis">

            
                                  <em>imagelist</em>

            
                                </span>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            		Same as <code class="option">lib-image:</code>, but the matching images

            
            		are excluded.

            
            		</p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">event:</code>

            
                                <span class="emphasis">

            
                                  <em>eventlist</em>

            
                                </span>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            		The symbolic event name to match on, e.g. <code class="option">event:DATA_MEM_REFS</code>.

            
            		You can pass a list of events for side-by-side comparison with <span class="command"><strong>opreport</strong></span>.

            
            		When using the timer interrupt, the event is always "TIMER".

            
            		</p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">count:</code>

            
                                <span class="emphasis">

            
                                  <em>eventcountlist</em>

            
                                </span>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            		The event count to match on, e.g. <code class="option">event:DATA_MEM_REFS count:30000</code>.

            
            		Note that this value refers to the count value in the event spec you passed

            
            		to <span class="command"><strong>opcontrol</strong></span> or <span class="command"><strong>operf</strong></span> when setting up to do a

            
            		profile run.  It has nothing to do with the sample counts in the profile data

            
            		itself.

            
            		You can pass a list of events for side-by-side comparison with <span class="command"><strong>opreport</strong></span>.

            
            		When using the timer interrupt, the count is always 0 (indicating it cannot be set).

            
            		</p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">unit-mask:</code>

            
                                <span class="emphasis">

            
                                  <em>masklist</em>

            
                                </span>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            		The unit mask value of the event to match on, e.g. <code class="option">unit-mask:1</code>.

            
            		You can pass a list of events for side-by-side comparison with <span class="command"><strong>opreport</strong></span>.

            
            		</p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">cpu:</code>

            
                                <span class="emphasis">

            
                                  <em>cpulist</em>

            
                                </span>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            		Only consider profiles for the given numbered CPU (starting from zero).

            
            		This is only useful when using CPU profile separation.

            
            		</p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">tgid:</code>

            
                                <span class="emphasis">

            
                                  <em>pidlist</em>

            
                                </span>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            		Only consider profiles for the given task groups. Unless some program

            
            		is using threads, the task group ID of a process is the same

            
            		as its process ID. This option corresponds to the POSIX

            
            		notion of a thread group.

            
            		This is only useful when using per-process profile separation.

            
            		</p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">tid:</code>

            
                                <span class="emphasis">

            
                                  <em>tidlist</em>

            
                                </span>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            		Only consider profiles for the given threads. When using

            
            		recent thread libraries, all threads in a process share the

            
            		same task group ID, but have different thread IDs. You can

            
            		use this option in combination with <code class="option">tgid:</code> to

            
            		restrict the results to particular threads within a process.

            
            		This is only useful when using per-process profile separation.

            
            		</p>

            
                            </dd>

            
                          </dl>

            
                        </div>

            
                      </div>

            
                      <div class="sect2" title="1.3. Locating and managing binary images">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="locating-and-managing-binary-images"></a>1.3. Locating and managing binary images</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            Each session's sample files can be found in the $SESSION_DIR/samples/ directory (default when

            
            using legacy mode: <code class="filename">/var/lib/oprofile/samples/</code>; default when using

            
            <span class="command"><strong>operf</strong></span>:  <code class="filename">&lt;cur_dir&gt;/oprofile_data/samples/</code>).

            
            These are used, along with the binary image files, to produce human-readable data.

            
            In some circumstances (e.g., kernel modules), OProfile

            
            will not be able to find the binary images. All the tools have an <code class="option">--image-path</code>

            
            option to which you can pass a comma-separated list of alternate paths to search. For example,

            
            I can let OProfile find my 2.6 modules by using <span class="command"><strong>--image-path /lib/modules/2.6.0/kernel/</strong></span>.

            
            It is your responsibility to ensure that the correct images are found when using this

            
            option.

            
            </p>

            
                        <p>

            
            Note that if a binary image changes after the sample file was created, you won't be able to get useful

            
            symbol-based data out. This situation is detected for you. If you replace a binary, you should

            
            make sure to save the old binary if you need to do comparative profiles.

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="1.4. What to do when you don't get any results">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="no-results"></a>1.4. What to do when you don't get any results</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            When attempting to get output, you may see the error :

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
            error: no sample files found: profile specification too strict ?

            
            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            What this is saying is that the profile specification you passed in,

            
            when matched against the available sample files, resulted in no matches.

            
            There are a number of reasons this might happen:

            
            </p>

            
                        <div class="variablelist">

            
                          <dl>

            
                            <dt>

            
                              <span class="term">spelling</span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            You specified a binary name, but spelt it wrongly. Check your spelling !

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">profiler wasn't running</span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Make very sure that OProfile was actually up and running when you ran

            
            the application you wish to profile.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">application didn't run long enough</span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Remember OProfile is a statistical profiler - you're not guaranteed to

            
            get samples for short-running programs. You can help this by using a

            
            lower count for the performance counter, so there are a lot more samples

            
            taken per second.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">application spent most of its time in libraries</span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Similarly, if the application spends little time in the main binary image

            
            itself, with most of it spent in shared libraries it uses, you might

            
            not see any samples for the binary image (i.e., executable) itself. If you're

            
            using OProfile legacy mode profiling, then we recommend using

            
            <span class="command"><strong>opcontrol --separate=lib</strong></span> before the

            
            profiling session so that <span class="command"><strong>opreport</strong></span> and friends show

            
            the library profiles on a per-application basis.  This is done automatically

            
            when profiling with <span class="command"><strong>operf</strong></span>, so no special setup is necessary.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">specification was really too strict</span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            For example, you specified something like <code class="option">tgid:3433</code>,

            
            but no task with that group ID ever ran the code.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">application didn't generate any events</span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            If you're using a particular event counter, for example counting MMX

            
            operations, the code might simply have not generated any events in the

            
            first place. Verify the code you're profiling does what you expect it

            
            to.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">you didn't specify kernel module name correctly</span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            If you're trying to get reports for a kernel

            
            module, make sure to use the <code class="option">-p</code> option, and specify the

            
            module name <span class="emphasis"><em>with</em></span> the <code class="filename">.ko</code>

            
            extension. Check if the module is one loaded from initrd.

            
            </p>

            
                            </dd>

            
                          </dl>

            
                        </div>

            
                      </div>

            
                    </div>

            
                    <div class="sect1" title="2. Image summaries and symbol summaries (opreport)">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="opreport"></a>2. Image summaries and symbol summaries (<span class="command"><strong>opreport</strong></span>)</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            The <span class="command"><strong>opreport</strong></span> utility is the primary utility you will use for

            
            getting formatted data out of OProfile. It produces two types of data: image summaries

            
            and symbol summaries. An image summary lists the number of samples for individual

            
            binary images such as libraries or applications. Symbol summaries provide per-symbol

            
            profile data. In the following example, we're getting an image summary for the whole

            
            system:

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            $ opreport --long-filenames

            
            CPU: PIII, speed 863.195 MHz (estimated)

            
            Counted CPU_CLK_UNHALTED events (clocks processor is not halted) with a unit mask of 0x00 (No unit mask) count 23150

            
               905898 59.7415 /usr/lib/gcc-lib/i386-redhat-linux/3.2/cc1plus

            
               214320 14.1338 /boot/2.6.0/vmlinux

            
               103450  6.8222 /lib/i686/libc-2.3.2.so

            
                60160  3.9674 /usr/local/bin/madplay

            
                31769  2.0951 /usr/local/oprofile-pp/bin/oprofiled

            
                26550  1.7509 /usr/lib/libartsflow.so.1.0.0

            
                23906  1.5765 /usr/bin/as

            
                18770  1.2378 /oprofile

            
                15528  1.0240 /usr/lib/qt-3.0.5/lib/libqt-mt.so.3.0.5

            
                11979  0.7900 /usr/X11R6/bin/XFree86

            
                11328  0.7471 /bin/bash

            
                ...

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            If we had specified <code class="option">--symbols</code> in the previous command, we would have

            
            gotten a symbol summary of all the images across the entire system. We can restrict this to only

            
            part of the system profile; for example,

            
            below is a symbol summary of the OProfile daemon. Note that as we used

            
            <span class="command"><strong>opcontrol --separate=lib,kernel</strong></span>, symbols from images that <span class="command"><strong>oprofiled</strong></span>

            
            has used are also shown.

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            $ opreport -l -p /lib/modules/`uname -r` `which oprofiled` 2&gt;/dev/null | more

            
            CPU: Core 2, speed 2.534e+06 MHz (estimated)

            
            Counted CPU_CLK_UNHALTED events (Clock cycles when not halted) with a unit mask of 0x00 (Unhalted core cycles) count 100000

            
            samples  %        image name               symbol name

            
            1353     24.9447  vmlinux                  sidtab_context_to_sid

            
            500       9.2183  vmlinux                  avtab_hash_eval

            
            154       2.8392  vmlinux                  __link_path_walk

            
            152       2.8024  vmlinux                  d_prune_aliases

            
            120       2.2124  vmlinux                  avtab_search_node

            
            104       1.9174  vmlinux                  find_next_bit

            
            85        1.5671  vmlinux                  selinux_file_fcntl

            
            82        1.5118  vmlinux                  avtab_write

            
            81        1.4934  oprofiled                odb_update_node_with_offset

            
            73        1.3459  oprofiled                opd_process_samples

            
            72        1.3274  vmlinux                  avc_has_perm_noaudit

            
            61        1.1246  libc-2.12.so             _IO_vfscanf

            
            59        1.0878  ext4.ko                  ext4_mark_iloc_dirty

            
            ...

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            These are the two basic ways you are most likely to use regularly, but <span class="command"><strong>opreport</strong></span>

            
            can do a lot more than that, as described below.

            
            </p>

            
                      <div class="sect2" title="2.1. Merging separate profiles"><div class="titlepage"><div><div><h3 class="title"><a id="opreport-merging"></a>2.1. Merging separate profiles</h3></div></div></div>


            If you have used one of the <code class="option">--separate[*]</code> options

            
            whilst profiling, there can be several separate profiles for

            
            a single binary image within a session. Normally the output

            
            will keep these images separated. So, for example, if you profiled

            
            with separation on a per-cpu basis (<code class="code">opcontrol --separate=cpu</code> or

            
            <code class="code">operf --separate-cpu</code>), you would see separate columns in

            
            the output of <span class="command"><strong>opreport</strong></span> for each CPU where samples

            
            were recorded. But it can be useful to merge these results back together

            
            to make the report more readable. The <code class="option">--merge</code> option allows

            
            you to do that.

            
            </div>

            
                      <div class="sect2" title="2.2. Side-by-side multiple results"><div class="titlepage"><div><div><h3 class="title"><a id="opreport-comparison"></a>2.2. Side-by-side multiple results</h3></div></div></div>

            
            If you have used multiple events when profiling, by default you get

            
            side-by-side results of each event's sample values from <span class="command"><strong>opreport</strong></span>.

            
            You can restrict which events to list by appropriate use of the

            
            <code class="option">event:</code> profile specifications, etc.

            
            </div>

            
                      <div class="sect2" title="2.3. Callgraph output">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="opreport-callgraph"></a>2.3. Callgraph output</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            This section provides details on how to use the OProfile callgraph feature.

            
            </p>

            
                        <div class="sect3" title="2.3.1. Callgraph details">

            
                          <div class="titlepage">

            
                            <div>

            
                              <div>

            
                                <h4 class="title"><a id="op-cg1"></a>2.3.1. Callgraph details</h4>

            
                              </div>

            
                            </div>

            
                          </div>

            
                          <p>

            
            When using the <code class="option">--callgraph</code> option, you can see what

            
            functions are calling other functions in the output. Consider the

            
            following program:

            
            </p>

            
                          <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                            <tr>

            
                              <td>

            
                                <pre class="screen">

            
            #include &lt;string.h&gt;

            
            #include &lt;stdlib.h&gt;

            
            #include &lt;stdio.h&gt;


            #define SIZE 500000


            static int compare(const void *s1, const void *s2)

            
            {

            
                    return strcmp(s1, s2);

            
            }


            static void repeat(void)

            
            {

            
                    int i;

            
                    char *strings[SIZE];

            
                    char str[] = "abcdefghijklmnopqrstuvwxyz";


                    for (i = 0; i &lt; SIZE; ++i) {

            
                            strings[i] = strdup(str);

            
                            strfry(strings[i]);

            
                    }


                    qsort(strings, SIZE, sizeof(char *), compare);

            
            }


            int main()

            
            {

            
                    while (1)

            
                            repeat();

            
            }

            
            </pre>

            
                              </td>

            
                            </tr>

            
                          </table>

            
                          <p>

            
            When running with the call-graph option, OProfile will

            
            record the function stack every time it takes a sample.

            
            <span class="command"><strong>opreport --callgraph</strong></span> outputs an entry for each

            
            function, where each entry looks similar to:

            
            </p>

            
                          <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                            <tr>

            
                              <td>

            
                                <pre class="screen">

            
            samples  %        image name               symbol name

            
              197       0.1548  cg                       main

            
              127036   99.8452  cg                       repeat

            
            84590    42.5084  libc-2.3.2.so            strfry

            
              84590    66.4838  libc-2.3.2.so            strfry [self]

            
              39169    30.7850  libc-2.3.2.so            random_r

            
              3475      2.7312  libc-2.3.2.so            __i686.get_pc_thunk.bx

            
            -------------------------------------------------------------------------------

            
            </pre>

            
                              </td>

            
                            </tr>

            
                          </table>

            
                          <p>

            
            Here the non-indented line is the function we're focussing upon

            
            (<code class="function">strfry()</code>). This

            
            line is the same as you'd get from a normal <span class="command"><strong>opreport</strong></span>

            
            output.

            
            </p>

            
                          <p>

            
            Above the non-indented line we find the functions that called this

            
            function (for example, <code class="function">repeat()</code> calls

            
            <code class="function">strfry()</code>). The samples and percentage values here

            
            refer to the number of times we took a sample where this call was found

            
            in the stack; the percentage is relative to all other callers of the

            
            function we're focussing on. Note that these values are

            
            <span class="emphasis"><em>not</em></span> call counts; they only reflect the call stack

            
            every time a sample is taken; that is, if a call is found in the stack

            
            at the time of a sample, it is recorded in this count.

            
            </p>

            
                          <p>

            
            Below the line are functions that are called by

            
            <code class="function">strfry()</code> (called <span class="emphasis"><em>callees</em></span>).

            
            It's clear here that <code class="function">strfry()</code> calls

            
            <code class="function">random_r()</code>. We also see a special entry with a

            
            "[self]" marker. This records the normal samples for the function, but

            
            the percentage becomes relative to all callees. This allows you to

            
            compare time spent in the function itself compared to functions it

            
            calls. Note that if a function calls itself, then it will appear in the

            
            list of callees of itself, but without the "[self]" marker; so recursive

            
            calls are still clearly separable.

            
            </p>

            
                          <p>

            
            You may have noticed that the output lists <code class="function">main()</code>

            
            as calling <code class="function">strfry()</code>, but it's clear from the source

            
            that this doesn't actually happen. See <a class="xref" href="#interpreting-callgraph" title="3. Interpreting call-graph profiles">Section 3, &#8220;Interpreting call-graph profiles&#8221;</a> for an explanation.

            
            </p>

            
                        </div>

            
                        <div class="sect3" title="2.3.2. Callgraph and JIT support">

            
                          <div class="titlepage">

            
                            <div>

            
                              <div>

            
                                <h4 class="title"><a id="cg-with-jitsupport"></a>2.3.2. Callgraph and JIT support</h4>

            
                              </div>

            
                            </div>

            
                          </div>

            
                          <p>

            
            Callgraph output where anonymously mapped code is in the callstack can sometimes be misleading.

            
            For all such code, the samples for the anonymously mapped code are stored in a samples subdirectory

            
            named <code class="filename">{anon:anon}/&lt;tgid&gt;.&lt;begin_addr&gt;.&lt;end_addr&gt;</code>.

            
            As stated earlier, if this anonymously mapped code is JITed code from a supported VM like Java,

            
            OProfile creates an ELF file to provide a (somewhat) permanent backing file for the code.

            
            However, when viewing callgraph output, any anonymously mapped code in the callstack

            
            will be attributed to <code class="filename">anon (&lt;tgid&gt;: range:&lt;begin_addr&gt;-&lt;end_addr&gt;</code>,

            
            even if a <code class="filename">.jo</code> ELF file had been created for it.  See the example below.

            
            </p>

            
                          <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                            <tr>

            
                              <td>

            
                                <pre class="screen">

            
            -------------------------------------------------------------------------------

            
              1         2.2727  libj9ute23.so            java.bin                 traceV

            
              2         4.5455  libj9ute23.so            java.bin                 utsTraceV

            
              4         9.0909  libj9trc23.so            java.bin                 fillInUTInterfaces

            
              37       84.0909  libj9trc23.so            java.bin                 twGetSequenceCounter

            
            8         0.0154  libj9prt23.so            java.bin                 j9time_hires_clock

            
              27       61.3636  anon (tgid:10014 range:0x100000-0x103000) java.bin                 (no symbols)

            
              9        20.4545  libc-2.4.so              java.bin                 gettimeofday

            
              8        18.1818  libj9prt23.so            java.bin                 j9time_hires_clock [self]

            
            -------------------------------------------------------------------------------

            
            </pre>

            
                              </td>

            
                            </tr>

            
                          </table>

            
                          <p>

            
            The output shows that "anon (tgid:10014 range:0x100000-0x103000)" was a callee of

            
            <code class="code">j9time_hires_clock</code>, even though the ELF file <code class="filename">10014.jo</code> was

            
            created for this profile run.  Unfortunately, there is currently no way to correlate

            
            that anonymous callgraph entry with its corresponding <code class="filename">.jo</code> file.

            
            </p>

            
                        </div>

            
                      </div>

            
                      <div class="sect2" title="2.4. Differential profiles with opreport">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="opreport-diff"></a>2.4. Differential profiles with <span class="command"><strong>opreport</strong></span></h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            Often, we'd like to be able to compare two profiles. For example, when

            
            analysing the performance of an application, we'd like to make code

            
            changes and examine the effect of the change. This is supported in

            
            <span class="command"><strong>opreport</strong></span> by giving a profile specification that

            
            identifies two different profiles. The general form is of:

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
            $ opreport &lt;shared-spec&gt; { &lt;first-profile&gt; } { &lt;second-profile&gt; }

            
            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;">

            
                          <h3 class="title">Note</h3>

            
                          <p>

            
            We lost our Dragon book down the back of the sofa, so you have to be

            
            careful to have spaces around those braces, or things will get

            
            hopelessly confused. We can only apologise.

            
            </p>

            
                        </div>

            
                        <p>

            
            For each of the profiles, the shared section is prefixed, and then the

            
            specification is analysed. The usual parameters work both within the

            
            shared section, and in the sub-specification within the curly braces.

            
            </p>

            
                        <p>

            
            A typical way to use this feature is with archives created with

            
            <span class="command"><strong>oparchive</strong></span>. Let's look at an example:

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
            $ ./a

            
            $ oparchive -o orig ./a

            
            $ opcontrol --reset

            
              # edit and recompile a

            
            $ ./a

            
              # now compare the current profile of a with the archived profile

            
            $ opreport -xl ./a { archive:./orig } { }

            
            CPU: PIII, speed 863.233 MHz (estimated)

            
            Counted CPU_CLK_UNHALTED events (clocks processor is not halted) with a

            
            unit mask of 0x00 (No unit mask) count 100000

            
            samples  %        diff %    symbol name

            
            92435    48.5366  +0.4999   a

            
            54226    ---      ---       c

            
            49222    25.8459  +++       d

            
            48787    25.6175  -2.2e-01  b

            
            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            Note that we specified an empty second profile in the curly braces, as

            
            we wanted to use the current session; alternatively, we could

            
            have specified another archive, or a tgid etc. We specified the binary

            
            <span class="command"><strong>a</strong></span> in the shared section, so we matched that in both

            
            the profiles we're diffing.

            
            </p>

            
                        <p>

            
            As in the normal output, the results are sorted by the number of

            
            samples, and the percentage field represents the relative percentage of

            
            the symbol's samples in the second profile.

            
            </p>

            
                        <p>

            
            Notice the new column in the output. This value represents the

            
            percentage change of the relative percent between the first and the

            
            second profile: roughly, "how much more important this symbol is".

            
            Looking at the symbol <code class="function">a()</code>, we can see that it took

            
            roughly the same amount of the total profile in both the first and the

            
            second profile. The function <code class="function">c()</code> was not in the new

            
            profile, so has been marked with <code class="function">---</code>. Note that the

            
            sample value is the number of samples in the first profile; since we're

            
            displaying results for the second profile, we don't list a percentage

            
            value for it, as it would be meaningless. <code class="function">d()</code> is

            
            new in the second profile, and consequently marked with

            
            <code class="function">+++</code>.

            
            </p>

            
                        <p>

            
            When comparing profiles between different binaries, it should be clear

            
            that functions can change in terms of VMA and size. To avoid this

            
            problem, <span class="command"><strong>opreport</strong></span> considers a symbol to be the same

            
            if the symbol name, image name, and owning application name all match;

            
            any other factors are ignored. Note that the check for application name

            
            means that trying to compare library profiles between two different

            
            applications will not work as you might expect: each symbol will be

            
            considered different.

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="2.5. Anonymous executable mappings">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="opreport-anon"></a>2.5. Anonymous executable mappings</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            Many applications, typically ones involving dynamic compilation into

            
            machine code (just-in-time, or "JIT", compilation), have executable mappings that

            
            are not backed by an ELF file. <span class="command"><strong>opreport</strong></span> has basic support for showing the

            
            samples taken in these regions; for example:

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
            $ opreport /usr/bin/mono -l

            
            CPU: ppc64 POWER5, speed 1654.34 MHz (estimated)

            
            Counted CYCLES events (Processor Cycles using continuous sampling) with a unit mask of 0x00 (No unit mask) count 100000

            
            samples  %        image name    		                symbol name

            
            47       58.7500  mono                     			(no symbols)

            
            14       17.5000  anon (tgid:3189 range:0xf72aa000-0xf72fa000)  (no symbols)

            
            9        11.2500  anon (tgid:3189 range:0xf6cca000-0xf6dd9000)  (no symbols)

            
            .	 .	  .						.

            
            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            </p>

            
                        <p>

            
            Note that, since such mappings are dependent upon individual invocations of

            
            a binary, these mappings are always listed as a dependent image,

            
            even when using the legacy mode <code class="option">opcontrol --separate=none</code> command.

            
            Equally, the results are not affected by the <code class="option">--merge</code>

            
            option.

            
            </p>

            
                        <p>

            
            As shown in the opreport output above, OProfile is unable to attribute the samples to any

            
            symbol(s) because there is no ELF file for this code.

            
            Enhanced support for JITed code is now available for some virtual machines;

            
            e.g., the Java Virtual Machine.  For details about OProfile output for

            
            JITed code, see <a class="xref" href="#getting-jit-reports" title="4. OProfile results with JIT samples">Section 4, &#8220;OProfile results with JIT samples&#8221;</a>.

            
            </p>

            
                        <p>For more information about JIT support in OProfile, see <a class="xref" href="#jitsupport" title="4.1. Support for dynamically compiled (JIT) code">Section 4.1, &#8220;Support for dynamically compiled (JIT) code&#8221;</a>.

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="2.6. XML formatted output">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="opreport-xml"></a>2.6. XML formatted output</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            The --xml option can be used to generate XML instead of the usual

            
            text format.  This allows opreport to eliminate some of the constraints

            
            dictated by the two dimensional text format.  For example, it is possible

            
            to separate the sample data across multiple events, cpus and threads.  The XML

            
            schema implemented by opreport is found in doc/opreport.xsd. It contains

            
            more detailed comments about the structure of the XML generated by opreport.

            
            </p>

            
                        <p>

            
            Since XML is consumed by a client program rather than a user, its structure

            
            is fairly static.  In particular, the --sort option is incompatible with the

            
            --xml option.  Percentages are not dislayed in the XML so the options related

            
            to percentages will have no effect.  Full pathnames are always displayed in

            
            the XML so --long-filenames is not necessary.  The --details option will cause

            
            all of the individual sample data to be included in the XML as well as the

            
            instruction byte stream for each symbol (for doing disassembly) and can result

            
            in very large XML files.

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="2.7. Options for opreport">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="opreport-options"></a>2.7. Options for <span class="command"><strong>opreport</strong></span></h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <div class="variablelist">

            
                          <dl>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--accumulated / -a</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Accumulate sample and percentage counts in the symbol list.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--callgraph / -c</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Show callgraph information.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--debug-info / -g</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Show source file and line for each symbol.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--demangle / -D none|normal|smart</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            none: no demangling. normal: use default demangler (default) smart: use

            
            pattern-matching to make C++ symbol demangling more readable.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--details / -d</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Show per-instruction details for all selected symbols. Note that, for

            
            binaries without symbol information, the VMA values shown are raw file

            
            offsets for the image binary.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--exclude-dependent / -x</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Do not include application-specific images for libraries, kernel modules

            
            and the kernel. This option only makes sense if the profile session

            
            used --separate.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--exclude-symbols / -e [symbols]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Exclude all the symbols in the given comma-separated list.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--global-percent / -%</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Make all percentages relative to the whole profile.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--help / -? / --usage</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Show help message.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--image-path / -p [paths]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Comma-separated list of additional paths to search for binaries.

            
            This is needed to find kernel modules.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--root / -R [path]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            A path to a filesystem to search for additional binaries.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--include-symbols / -i [symbols]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Only include symbols in the given comma-separated list.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--long-filenames / -f</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Output full paths instead of basenames.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--merge / -m [lib,cpu,tid,tgid,unitmask,all]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Merge any profiles separated in a --separate session.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--no-header</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Don't output a header detailing profiling parameters.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--output-file / -o [file]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Output to the given file instead of stdout.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--reverse-sort / -r</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Reverse the sort from the default.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term"><code class="option">--session-dir=</code>dir_path</span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Use sample database out of directory <code class="filename">dir_path</code>

            
            instead of the default location (/var/lib/oprofile).

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--show-address / -w</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Show the VMA address of each symbol (off by default).

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--sort / -s [vma,sample,symbol,debug,image]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Sort the list of symbols by, respectively, symbol address,

            
            number of samples, symbol name, debug filename and line number,

            
            binary image filename.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--symbols / -l</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            List per-symbol information instead of a binary image summary.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--threshold / -t [percentage]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Only output data for symbols that have more than the given percentage

            
            of total samples.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--verbose / -V [options]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Give verbose debugging output.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--version / -v</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Show version.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--xml / -X</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Generate XML output.

            
            </p>

            
                            </dd>

            
                          </dl>

            
                        </div>

            
                      </div>

            
                    </div>

            
                    <div class="sect1" title="3. Outputting annotated source (opannotate)">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="opannotate"></a>3. Outputting annotated source (<span class="command"><strong>opannotate</strong></span>)</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            The <span class="command"><strong>opannotate</strong></span> utility generates annotated source files or assembly listings, optionally

            
            mixed with source.

            
            If you want to see the source file, the profiled application needs to have debug information, and the source

            
            must be available through this debug information. For GCC, you must use the <code class="option">-g</code> option

            
            when you are compiling.

            
            If the binary doesn't contain sufficient debug information, you can still

            
            use <span class="command"><strong>opannotate <code class="option">--assembly</code></strong></span> to get annotated assembly

            
            as long as the binary has (at least) symbol information.

            
            </p>

            
                      <p>

            
            Note that for the reason explained in <a class="xref" href="#hardware-counters" title="5.1. Hardware performance counters">Section 5.1, &#8220;Hardware performance counters&#8221;</a> the results can be

            
            inaccurate. The debug information itself can add other problems; for example, the line number for a symbol can be

            
            incorrect. Assembly instructions can be re-ordered and moved by the compiler, and this can lead to

            
            crediting source lines with samples not really "owned" by this line. Also see

            
            <a class="xref" href="#interpreting" title="Chapter 5. Interpreting profiling results">Chapter 5, <i>Interpreting profiling results</i></a>.

            
            </p>

            
                      <p>

            
            You can output the annotation to one single file, containing all the source found using the

            
            <code class="option">--source</code>. You can use this in conjunction with <code class="option">--assembly</code>

            
            to get combined source/assembly output.

            
            </p>

            
                      <p>

            
            You can also output a directory of annotated source files that maintains the structure of

            
            the original sources. Each line in the annotated source is prepended with the samples

            
            for that line. Additionally, each symbol is annotated giving details for the symbol

            
            as a whole. An example:

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            $ opannotate --source --output-dir=annotated /usr/local/oprofile-pp/bin/oprofiled

            
            $ ls annotated/home/moz/src/oprofile-pp/daemon/

            
            opd_cookie.h  opd_image.c  opd_kernel.c  opd_sample_files.c  oprofiled.c

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            Line numbers are maintained in the source files, but each file has

            
            a footer appended describing the profiling details. The actual annotation

            
            looks something like this :

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            ...

            
                           :static uint64_t pop_buffer_value(struct transient * trans)

            
             11510  1.9661 :{ /* pop_buffer_value total:  89901 15.3566 */

            
                           :        uint64_t val;

            
                           :

            
             10227  1.7469 :        if (!trans-&gt;remaining) {

            
                           :                fprintf(stderr, "BUG: popping empty buffer !\n");

            
                           :                exit(EXIT_FAILURE);

            
                           :        }

            
                           :

            
                           :        val = get_buffer_value(trans-&gt;buffer, 0);

            
              2281  0.3896 :        trans-&gt;remaining--;

            
              2296  0.3922 :        trans-&gt;buffer += kernel_pointer_size;

            
                           :        return val;

            
             10454  1.7857 :}

            
            ...

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            The first number on each line is the number of samples, whilst the second is

            
            the relative percentage of total samples.

            
            </p>

            
                      <div class="sect2" title="3.1. Locating source files">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="opannotate-finding-source"></a>3.1. Locating source files</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            Of course, <span class="command"><strong>opannotate</strong></span> needs to be able to locate the source files

            
            for the binary image(s) in order to produce output. Some binary images have debug

            
            information where the given source file paths are relative, not absolute. You can

            
            specify search paths to look for these files (similar to <span class="command"><strong>gdb</strong></span>'s

            
            <code class="option">dir</code> command) with the <code class="option">--search-dirs</code> option.

            
            </p>

            
                        <p>

            
            Sometimes you may have a binary image which gives absolute paths for the source files,

            
            but you have the actual sources elsewhere (commonly, you've installed an SRPM for

            
            a binary on your system and you want annotation from an existing profile). You can

            
            use the <code class="option">--base-dirs</code> option to redirect OProfile to look somewhere

            
            else for source files. For example, imagine we have a binary generated from a source

            
            file that is given in the debug information as <code class="filename">/tmp/build/libfoo/foo.c</code>,

            
            and you have the source tree matching that binary installed in <code class="filename">/home/user/libfoo/</code>.

            
            You can redirect OProfile to find <code class="filename">foo.c</code> correctly like this :

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
            $ opannotate --source --base-dirs=/tmp/build/libfoo/ --search-dirs=/home/user/libfoo/ --output-dir=annotated/ /lib/libfoo.so

            
            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            You can specify multiple (comma-separated) paths to both options.

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="3.2. Usage of opannotate">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="opannotate-details"></a>3.2. Usage of <span class="command"><strong>opannotate</strong></span></h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <div class="variablelist">

            
                          <dl>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--assembly / -a</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Output annotated assembly. If this is combined with --source, then mixed

            
            source / assembly annotations are output.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--base-dirs / -b [paths]/</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Comma-separated list of path prefixes. This can be used to point OProfile to a

            
            different location for source files when the debug information specifies an

            
            absolute path on your system for the source that does not exist. The prefix

            
            is stripped from the debug source file paths, then searched in the search dirs

            
            specified by <code class="option">--search-dirs</code>.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--demangle / -D none|normal|smart</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            none: no demangling. normal: use default demangler (default) smart: use

            
            pattern-matching to make C++ symbol demangling more readable.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--exclude-dependent / -x</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Do not include application-specific images for libraries, kernel modules

            
            and the kernel. This option only makes sense if the profile session

            
            used --separate.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--exclude-file [files]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Exclude all files in the given comma-separated list of glob patterns.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--exclude-symbols / -e [symbols]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Exclude all the symbols in the given comma-separated list.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--help / -? / --usage</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Show help message.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--image-path / -p [paths]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Comma-separated list of additional paths to search for binaries.

            
            This is needed to find kernel modules.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--root / -R [path]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            A path to a filesystem to search for additional binaries.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--include-file [files]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Only include files in the given comma-separated list of glob patterns.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--include-symbols / -i [symbols]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Only include symbols in the given comma-separated list.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--objdump-params [params]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Pass the given parameters as extra values when calling objdump.

            
            If more than one option is to be passed to objdump, the parameters must be enclosed in a

            
            quoted string.

            
            </p>

            
                              <p>

            
            An example of where this option is useful is when your toolchain does not

            
            automatically recognize instructions that are specific to your processor.

            
            For example, on IBM POWER7/RHEL 6, objdump must be told that a binary file may have

            
            POWER7-specific instructions. The <span class="command"><strong>opannotate</strong></span> option to show the POWER7-specific

            
            instructions is:

            
            </p>

            
                              <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                                <tr>

            
                                  <td>

            
                                    <pre class="screen">

            
               --objdump-params=-Mpower7

            
            </pre>

            
                                  </td>

            
                                </tr>

            
                              </table>

            
                              <p>

            
            </p>

            
                              <p>

            
            The <span class="command"><strong>opannotate</strong></span> option to show the POWER7-specific instructions,

            
            the source code (--source) and the line numbers (-l) would be:

            
            </p>

            
                              <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                                <tr>

            
                                  <td>

            
                                    <pre class="screen">

            
               --objdump-params="-Mpower7 -l --source"

            
            </pre>

            
                                  </td>

            
                                </tr>

            
                              </table>

            
                              <p>

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--output-dir / -o [dir]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Output directory. This makes opannotate output one annotated file for each

            
            source file. This option can't be used in conjunction with --assembly.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--search-dirs / -d [paths]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Comma-separated list of paths to search for source files. This is useful to find

            
            source files when the debug information only contains relative paths.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--source / -s</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Output annotated source. This requires debugging information to be available

            
            for the binaries.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--threshold / -t [percentage]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Only output data for symbols that have more than the given percentage

            
            of total samples.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--verbose / -V [options]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Give verbose debugging output.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--version / -v</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Show version.

            
            </p>

            
                            </dd>

            
                          </dl>

            
                        </div>

            
                      </div>

            
                    </div>

            
                    <div class="sect1" title="4. OProfile results with JIT samples">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="getting-jit-reports"></a>4. OProfile results with JIT samples</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            		After profiling a Java (or other supported VM) application, the

            
            		OProfile JIT support creates ELF binaries from the

            
            		intermediate files that were written by the agent library.

            
            		The ELF binaries are named <code class="filename">&lt;tgid&gt;.jo</code>.

            
            		With the symbol information stored in these ELF files, it is

            
            		possible to map samples to the appropriate symbols.

            
            	</p>

            
                      <p>

            
            		The usual analysis tools (<span class="command"><strong>opreport</strong></span> and/or

            
            		<span class="command"><strong>opannotate</strong></span>) can now be used

            
            		to get symbols and assembly code for the instrumented VM processes.

            
            	</p>

            
                      <p>

            
            Below is an example of a profile report of a Java application that has been

            
            instrumented with the provided agent library.

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            $ opreport -l /usr/lib/jvm/jre-1.5.0-ibm/bin/java

            
            CPU: Core Solo / Duo, speed 2167 MHz (estimated)

            
            Counted CPU_CLK_UNHALTED events (Unhalted clock cycles) with a unit mask of 0x00 (Unhalted core cycles) count 100000

            
            samples  %        image name               symbol name

            
            186020   50.0523  no-vmlinux               no-vmlinux               (no symbols)

            
            34333     9.2380  7635.jo                  java                     void test.f1()

            
            19022     5.1182  libc-2.5.so              libc-2.5.so              _IO_file_xsputn@@GLIBC_2.1

            
            18762     5.0483  libc-2.5.so              libc-2.5.so              vfprintf

            
            16408     4.4149  7635.jo                  java                     void test$HelloThread.run()

            
            16250     4.3724  7635.jo                  java                     void test$test_1.f2(int)

            
            15303     4.1176  7635.jo                  java                     void test.f2(int, int)

            
            13252     3.5657  7635.jo                  java                     void test.f2(int)

            
            5165      1.3897  7635.jo                  java                     void test.f4()

            
            955       0.2570  7635.jo                  java                     void test$HelloThread.run()~


            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            </p>

            
                      <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;">

            
                        <h3 class="title">Note</h3>

            
                        <p>

            
            	  Depending on the JVM that is used, certain options of opreport and opannotate

            
            	  do NOT work since they rely on debug information (e.g. source code line number)

            
            	  that is not always available. The Sun JVM does provide the necessary debug

            
            	  information via the JVMTI[PI] interface,

            
            	  but other JVMs do not.

            
              </p>

            
                      </div>

            
                      <p>

            
            		As you can see in the opreport output, the JIT support agent for Java

            
            		generates symbols to include the class and method signature.

            
            		A symbol with the suffix &#732;&lt;n&gt; (e.g.

            
            		<code class="code">void test$HelloThread.run()&#732;1</code>) means that this is

            
            		the &lt;n&gt;th occurrence of the identical name. This happens if a method is re-JITed.

            
            		A symbol with the suffix %&lt;n&gt;, means that the address space of this symbol

            
            		was reused during the sample session (see <a class="xref" href="#overlapping-symbols" title="6. Overlapping symbols in JITed code">Section 6, &#8220;Overlapping symbols in JITed code&#8221;</a>).

            
            		The value &lt;n&gt; is the percentage of time that this symbol/code was present in

            
            		relation to the total lifetime of all overlapping other symbols. A symbol of the form

            
            		<code class="code">&lt;return_val&gt; &lt;class_name&gt;$&lt;method_sig&gt;</code> denotes an

            
            		inner class.

            
            	</p>

            
                    </div>

            
                    <div class="sect1" title="5. gprof-compatible output (opgprof)">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="opgprof"></a>5. <span class="command"><strong>gprof</strong></span>-compatible output (<span class="command"><strong>opgprof</strong></span>)</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            If you're familiar with the output produced by <span class="command"><strong>GNU gprof</strong></span>,

            
            you may find <span class="command"><strong>opgprof</strong></span> useful. It takes a single binary

            
            as an argument, and produces a <code class="filename">gmon.out</code> file for use

            
            with <span class="command"><strong>gprof -p</strong></span>. If call-graph profiling is enabled,

            
            then this is also included.

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            $ opgprof `which oprofiled` # generates gmon.out file

            
            $ gprof -p `which oprofiled` | head

            
            Flat profile:


            Each sample counts as 1 samples.

            
              %   cumulative   self              self     total

            
             time   samples   samples    calls  T1/call  T1/call  name

            
             33.13 206237.00 206237.00                             odb_insert

            
             22.67 347386.00 141149.00                             pop_buffer_value

            
              9.56 406881.00 59495.00                             opd_put_sample

            
              7.34 452599.00 45718.00                             opd_find_image

            
              7.19 497327.00 44728.00                             opd_process_samples

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <div class="sect2" title="5.1. Usage of opgprof">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="opgprof-details"></a>5.1. Usage of <span class="command"><strong>opgprof</strong></span></h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <div class="variablelist">

            
                          <dl>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--help / -? / --usage</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Show help message.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--image-path / -p [paths]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Comma-separated list of additional paths to search for binaries.

            
            This is needed to find kernel modules.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--root / -R [path]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            A path to a filesystem to search for additional binaries.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--output-filename / -o [file]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Output to the given file instead of the default, gmon.out

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--threshold / -t [percentage]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Only output data for symbols that have more than the given percentage

            
            of total samples.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--verbose / -V [options]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Give verbose debugging output.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--version / -v</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Show version.

            
            </p>

            
                            </dd>

            
                          </dl>

            
                        </div>

            
                      </div>

            
                    </div>

            
                    <div class="sect1" title="6. Analyzing profile data on another system (oparchive)">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="oparchive"></a>6. Analyzing profile data on another system (<span class="command"><strong>oparchive</strong></span>)</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            	The <span class="command"><strong>oparchive</strong></span> utility generates a directory populated

            
            	with executable, debug, and oprofile sample files. This directory can be

            
            	copied to another (host) machine and analyzed offline, with no further need to

            
            	access the data collection machine (target).

            
            </p>

            
                      <p>

            
            	The following command, executed on the target system, will collect the

            
            	sample files, the executables associated with the sample files, and the

            
            	debuginfo files associated with the executables and copy them into

            
            	<code class="filename">/tmp/current_data</code>:

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            # oparchive -o /tmp/current_data

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            	When transferring archived profile data to a host machine for offline analysis,

            
            	you need to determine if the oprofile ABI format is compatible between the

            
            	target system and the host system; if it isn't, you must run the <span class="command"><strong>opimport</strong></span>

            
            	command to convert the target's sample data files to the format of your host system.

            
            	See <a class="xref" href="#opimport" title="7. Converting sample database files (opimport)">Section 7, &#8220;Converting sample database files (<span class="command"><strong>opimport</strong></span>)&#8221;</a> for more details.

            
            </p>

            
                      <p>

            
            	After your profile data is transferred to the host system and (if necessary)

            
            	you have run the <span class="command"><strong>opimport</strong></span> command to convert the file

            
            	format, you can now run the <span class="command"><strong>opreport</strong></span> and

            
            	<span class="command"><strong>opannotate</strong></span> commands.  However, you must provide an

            
            	"archive specification" to let these post-processing tools know where to find

            
            	of the profile data (sample files, executables, etc.); for example:

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            # opreport archive:/home/user1/my_oprofile_archive --symbols

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            	Furthermore, if your profile was collected on your target system into a session-dir

            
            	other than <code class="filename">/var/lib/oprofile</code>, the <span class="command"><strong>oparchive</strong></span>

            
            	command will display a message similar to the following:

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            # NOTE: The sample data in this archive is located at /home/user1/test-stuff/oprofile_data

            
            instead of the standard location of /var/lib/oprofile.  Hence, when using opreport

            
            and other post-processing tools on this archive, you must pass the following option:

            
                    --session-dir=/home/user1/test-stuff/oprofile_data

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            	Then the above <span class="command"><strong>opreport</strong></span> example would have to include that

            
            	<code class="option">--session-dir</code> option.

            
            </p>

            
                      <p>

            
            </p>

            
                      <div class="note" title="Note" style="margin-left: 0.5in; margin-right: 0.5in;"><h3 class="title">Note</h3>

            
            	 In some host/target development environments, all target executables, libraries, and

            
            	 debuginfo files are stored in a root directory on the host to facilitate offline

            
            	 analysis.  In such cases, the <span class="command"><strong>oparchive</strong></span> command collects more data

            
            	 than is necessary; so, when copying the resulting output of <span class="command"><strong>oparchive</strong></span>,

            
            	 you can skip all of the executables, etc, and just archive the <code class="filename">$SESSION_DIR</code>

            
            	 tree located within the output directory you specified in your <span class="command"><strong>oparchive</strong></span>

            
            	 command. Then, when running the <span class="command"><strong>opreport</strong></span> or	<span class="command"><strong>opannotate</strong></span>

            
            	 commands on your host system, pass the <code class="option">--root</code> option to point to the

            
            	 location of your target's executables, etc.

            
            </div>

            
                      <p>

            
            </p>

            
                      <div class="sect2" title="6.1. Usage of oparchive">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="oparchive-details"></a>6.1. Usage of <span class="command"><strong>oparchive</strong></span></h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <div class="variablelist">

            
                          <dl>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--help / -? / --usage</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Show help message.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--exclude-dependent / -x</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Do not include application-specific images for libraries, kernel modules

            
            and the kernel. This option only makes sense if the profile session

            
            used --separate.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--image-path / -p [paths]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Comma-separated list of additional paths to search for binaries.

            
            This is needed to find kernel modules.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--root / -R [path]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            A path to a filesystem to search for additional binaries.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--output-directory / -o [directory]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Output to the given directory. There is no default. This must be specified.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--list-files / -l</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Only list the files that would be archived, don't copy them.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--verbose / -V [options]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Give verbose debugging output.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--version / -v</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Show version.

            
            </p>

            
                            </dd>

            
                          </dl>

            
                        </div>

            
                      </div>

            
                    </div>

            
                    <div class="sect1" title="7. Converting sample database files (opimport)">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="opimport"></a>7. Converting sample database files (<span class="command"><strong>opimport</strong></span>)</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            	This utility converts sample database files from a foreign binary format (abi) to

            
            	the native format. This is required when moving sample files to a (host) system

            
            	other than the one used for collection (target system), and the host and target systems are different

            
            	architectures. The abi format of the sample files to be imported is described in a

            
            	text file located in <code class="filename">$SESSION_DIR/abi</code>.  If you are unsure if

            
            	your target and host systems have compatible architectures (in regard to the OProfile

            
            	ABI), simply diff a <code class="filename">$SESSION_DIR/abi</code> file from the target system

            
            	with one from the host system.  If any differences show up at all, you must run the

            
            	<span class="command"><strong>opimport</strong></span> command.

            
            </p>

            
                      <p>

            
            	The <span class="command"><strong>oparchive</strong></span> command should be used on the machine where

            
            	the profile was taken (target) in order to collect sample files and all other necessary

            
            	information. The archive directory that is the output from <span class="command"><strong>oparchive</strong></span>

            
            	should be copied to the system where you wish to perform your performance analysis (host).

            
            </p>

            
                      <p>

            
            	The following command converts an input sample file to the specified

            
            	output sample file using the given abi file as a binary description

            
            	of the input file and the curent platform abi as a binary description

            
            	of the output file. (NOTE: The ellipses are used to make the example more

            
            	compact and cannot be used in an actual command line.)

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            # opimport -a /tmp/foreign-abi -o /tmp/imported/.../GLOBAL_POWER_EVENTS.200000.1.all.all.all /tmp/archived/var/lib/.../mprime/GLOBAL_POWER_EVENTS.200000.1.all.all.all

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
                Since opimport converts just one file at a time, an example shell script is provided below

            
                that will perform an import/conversion of all sample files in a samples directory collected

            
                from the target system.

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            #!/bin/bash

            
            Usage: my-import.sh &lt;input-abi-pathname&gt;


            # NOTE: Start from the "samples" directory containing the "current" directory

            
            # to be imported


            mkdir current-imported

            
            cd current-imported; (cd ../current; find . -type d ! -name .) |xargs mkdir

            
            cd ../current; mv stats ../StatsSave; find . -type f | while read line; do opimport  -a $1 -o ../current-imported/$line $line; done; mv ../StatsSave stats;

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            </p>

            
                      <p>

            
            Example usage:  Assume that on the target system, a profile was collected using a session-dir of

            
            <code class="filename">/var/lib/oprofile</code>, and then <span class="command"><strong>oparchive -o profile1</strong></span> was run.

            
            Then the <code class="filename">profile1</code> directory is copied to the host system for analysis.  To import

            
            the sample data in <code class="filename">profile1</code>, you would perform the following steps:

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            $cd profile1/var/lib/oprofile/samples

            
            $my-import.sh `pwd`/../abi

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            </p>

            
                      <div class="sect2" title="7.1. Usage of opimport">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="opimport-details"></a>7.1. Usage of <span class="command"><strong>opimport</strong></span></h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <div class="variablelist">

            
                          <dl>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--help / -? / --usage</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Show help message.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--abi / -a [filename]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Input abi file description location.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--force / -f</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Force conversion even if the input and output abi are identical.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--output / -o [filename]</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Specify the output filename. If the output file already exists, the file is

            
            not overwritten but data are accumulated in. Sample filename are informative

            
            for post profile tools and must be kept identical, in other word the pathname

            
            from the first path component containing a '{' must be kept as it in the

            
            output filename.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--verbose / -V</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Give verbose debugging output.

            
            </p>

            
                            </dd>

            
                            <dt>

            
                              <span class="term">

            
                                <code class="option">--version / -v</code>

            
                              </span>

            
                            </dt>

            
                            <dd>

            
                              <p>

            
            Show version.

            
            </p>

            
                            </dd>

            
                          </dl>

            
                        </div>

            
                      </div>

            
                    </div>

            
                  </div>

            
                  <div class="chapter" title="Chapter 5. Interpreting profiling results">

            
                    <div class="titlepage">

            
                      <div>

            
                        <div>

            
                          <h2 class="title"><a id="interpreting"></a>Chapter 5. Interpreting profiling results</h2>

            
                        </div>

            
                      </div>

            
                    </div>

            
                    <div class="toc">

            
                      <p>

            
                        <b>Table of Contents</b>

            
                      </p>

            
                      <dl>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#irq-latency">1. Profiling interrupt latency</a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#kernel-profiling">2. Kernel profiling</a>

            
                          </span>

            
                        </dt>

            
                        <dd>

            
                          <dl>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#irq-masking">2.1. Interrupt masking</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#idle">2.2. Idle time</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#kernel-modules">2.3. Profiling kernel modules</a>

            
                              </span>

            
                            </dt>

            
                          </dl>

            
                        </dd>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#interpreting-callgraph">3. Interpreting call-graph profiles</a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#debug-info">4. Inaccuracies in annotated source</a>

            
                          </span>

            
                        </dt>

            
                        <dd>

            
                          <dl>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#effect-of-optimizations">4.1. Side effects of optimizations</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#prologues">4.2. Prologues and epilogues</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#inlined-function">4.3. Inlined functions</a>

            
                              </span>

            
                            </dt>

            
                            <dt>

            
                              <span class="sect2">

            
                                <a href="#wrong-linenr-info">4.4. Inaccuracy in line number information</a>

            
                              </span>

            
                            </dt>

            
                          </dl>

            
                        </dd>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#symbol-without-debug-info">5. Assembly functions</a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#overlapping-symbols">6. Overlapping symbols in JITed code</a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#interpreting_operf_results">7. Using operf to profile fork/execs</a>

            
                          </span>

            
                        </dt>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#hidden-cost">8. Other discrepancies</a>

            
                          </span>

            
                        </dt>

            
                      </dl>

            
                    </div>

            
                    <p>

            
            The standard caveats of profiling apply in interpreting the results from OProfile:

            
            profile realistic situations, profile different scenarios, profile

            
            for as long as a time as possible, avoid system-specific artifacts, don't trust

            
            the profile data too much. Also bear in mind the comments on the performance

            
            counters above - you <span class="emphasis"><em>cannot</em></span> rely on totally accurate

            
            instruction-level profiling.  However, for almost all circumstances the data

            
            can be useful. Ideally a utility such as Intel's VTUNE would be available to

            
            allow careful instruction-level analysis; go hassle Intel for this, not me ;)

            
            </p>

            
                    <div class="sect1" title="1. Profiling interrupt latency">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="irq-latency"></a>1. Profiling interrupt latency</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            This is an example of how the latency of delivery of profiling interrupts

            
            can impact the reliability of the profiling data. This is pretty much a

            
            worst-case-scenario example: these problems are fairly rare.

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            double fun(double a, double b, double c)

            
            {

            
             double result = 0;

            
             for (int i = 0 ; i &lt; 10000; ++i) {

            
              result += a;

            
              result *= b;

            
              result /= c;

            
             }

            
             return result;

            
            }

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            Here the last instruction of the loop is very costly, and you would expect the result

            
            reflecting that - but (cutting the instructions inside the loop):

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            $ opannotate -a -t 10 ./a.out


                 88 15.38% : 8048337:       fadd   %st(3),%st

            
                 48 8.391% : 8048339:       fmul   %st(2),%st

            
                 68 11.88% : 804833b:       fdiv   %st(1),%st

            
                368 64.33% : 804833d:       inc    %eax

            
                           : 804833e:       cmp    $0x270f,%eax

            
                           : 8048343:       jle    8048337

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            The problem comes from the x86 hardware; when the counter overflows the IRQ

            
            is asserted but the hardware has features that can delay the NMI interrupt:

            
            x86 hardware is synchronous (i.e. cannot interrupt during an instruction);

            
            there is also a latency when the IRQ is asserted, and the multiple

            
            execution units and the out-of-order model of modern x86 CPUs also causes

            
            problems. This is the same function, with annotation :

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            $ opannotate -s -t 10 ./a.out


                           :double fun(double a, double b, double c)

            
                           :{ /* _Z3funddd total:     572 100.0% */

            
                           : double result = 0;

            
                368 64.33% : for (int i = 0 ; i &lt; 10000; ++i) {

            
                 88 15.38% :  result += a;

            
                 48 8.391% :  result *= b;

            
                 68 11.88% :  result /= c;

            
                           : }

            
                           : return result;

            
                           :}

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            The conclusion: don't trust samples coming at the end of a loop,

            
            particularly if the last instruction generated by the compiler is costly. This

            
            case can also occur for branches. Always bear in mind that samples

            
            can be delayed by a few cycles from its real position. That's a hardware

            
            problem and OProfile can do nothing about it.

            
            </p>

            
                    </div>

            
                    <div class="sect1" title="2. Kernel profiling">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="kernel-profiling"></a>2. Kernel profiling</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <div class="sect2" title="2.1. Interrupt masking">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="irq-masking"></a>2.1. Interrupt masking</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            OProfile uses non-maskable interrupts (NMI) on the P6 generation, Pentium 4,

            
            Athlon, Opteron, Phenom, and Turion processors. These interrupts can occur even in sections of the

            
            kernel where interrupts are disabled, allowing collection of samples in virtually

            
            all executable code.  The timer interrupt mode and Itanium 2 collection mechanisms

            
            use maskable interrupts; therefore, these profiling mechanisms have "sample

            
            shadows", or blind spots: regions where no samples will be collected. Typically, the samples

            
            will be attributed to the code immediately after the interrupts are re-enabled.

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="2.2. Idle time">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="idle"></a>2.2. Idle time</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            Your kernel is likely to support halting the processor when a CPU is idle. As

            
            the typical hardware events like <code class="constant">CPU_CLK_UNHALTED</code> do not

            
            count when the CPU is halted, the kernel profile will not reflect the actual

            
            amount of time spent idle. You can change this behaviour by booting with

            
            the <code class="option">idle=poll</code> option, which uses a different idle routine. This

            
            will appear as <code class="function">poll_idle()</code> in your kernel profile.

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="2.3. Profiling kernel modules">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="kernel-modules"></a>2.3. Profiling kernel modules</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            OProfile profiles kernel modules by default. However, there are a couple of problems

            
            you may have when trying to get results. First, you may have booted via an initrd;

            
            this means that the actual path for the module binaries cannot be determined automatically.

            
            To get around this, you can use the <code class="option">-p</code> option to the profiling tools

            
            to specify where to look for the kernel modules.

            
            </p>

            
                        <p>

            
            In kernel version 2.6, the information on where kernel module binaries are located was removed.

            
            This means OProfile needs guiding with the <code class="option">-p</code> option to find your

            
            modules. Normally, you can just use your standard module top-level directory for this.

            
            Note that due to this problem, OProfile cannot check that the modification times match;

            
            it is your responsibility to make sure you do not modify a binary after a profile

            
            has been created.

            
            </p>

            
                        <p>

            
            If you have run <span class="command"><strong>insmod</strong></span> or <span class="command"><strong>modprobe</strong></span> to insert a module

            
            in a particular directory, it is important that you specify this directory with the

            
            <code class="option">-p</code> option first, so that it over-rides an older module binary that might

            
            exist in other directories you've specified with <code class="option">-p</code>. It is up to you

            
            to make sure that these values are correct: the kernel simply does not provide enough

            
            information for OProfile to get this information.

            
            </p>

            
                      </div>

            
                    </div>

            
                    <div class="sect1" title="3. Interpreting call-graph profiles">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="interpreting-callgraph"></a>3. Interpreting call-graph profiles</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            Sometimes the results from call-graph profiles may be different to what

            
            you expect to see. The first thing to check is whether the target

            
            binaries where compiled with frame pointers enabled (if the binary was

            
            compiled using <span class="command"><strong>gcc</strong></span>'s

            
            <code class="option">-fomit-frame-pointer</code> option, you will not get

            
            meaningful results). Note that as of this writing, the GCC developers

            
            plan to disable frame pointers by default. The Linux kernel is built

            
            without frame pointers by default; there is a configuration option you

            
            can use to turn it on under the "Kernel Hacking" menu.

            
            </p>

            
                      <p>

            
            Often you may see a caller of a function that does not actually directly

            
            call the function you're looking at (e.g. if <code class="function">a()</code>

            
            calls <code class="function">b()</code>, which in turn calls

            
            <code class="function">c()</code>, you may see an entry for

            
            <code class="function">a()-&gt;c()</code>).  What's actually occurring is that we

            
            are taking samples at the very start (or the very end) of

            
            <code class="function">c()</code>; at these few instructions, we haven't yet

            
            created the new function's frame, so it appears as if

            
            <code class="function">a()</code> is calling directly into

            
            <code class="function">c()</code>. Be careful not to be misled by these

            
            entries.

            
            </p>

            
                      <p>

            
            Like the rest of OProfile, call-graph profiling uses a statistical

            
            approach; this means that sometimes a backtrace sample is truncated, or

            
            even partially wrong. Bear this in mind when examining results.

            
            </p>

            
                    </div>

            
                    <div class="sect1" title="4. Inaccuracies in annotated source">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="debug-info"></a>4. Inaccuracies in annotated source</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <div class="sect2" title="4.1. Side effects of optimizations">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="effect-of-optimizations"></a>4.1. Side effects of optimizations</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            The compiler can introduce some pitfalls in the annotated source output.

            
            The optimizer can move pieces of code in such manner that two line of codes

            
            are interlaced (instruction scheduling). Also debug info generated by the compiler

            
            can show strange behavior. This is especially true for complex expressions e.g. inside

            
            an if statement:

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
            	if (a &amp;&amp; ..

            
            	    b &amp;&amp; ..

            
            	    c &amp;&amp;)

            
            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            here the problem come from the position of line number. The available debug

            
            info does not give enough details for the if condition, so all samples are

            
            accumulated at the position of the right brace of the expression. Using

            
            <span class="command"><strong>opannotate <code class="option">-a</code></strong></span> can help to show the real

            
            samples at an assembly level.

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="4.2. Prologues and epilogues">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="prologues"></a>4.2. Prologues and epilogues</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            The compiler generally needs to generate "glue" code across function calls, dependent

            
            on the particular function call conventions used. Additionally other things

            
            need to happen, like stack pointer adjustment for the local variables; this

            
            code is known as the function prologue. Similar code is needed at function return,

            
            and is known as the function epilogue. This will show up in annotations as

            
            samples at the very start and end of a function, where there is no apparent

            
            executable code in the source.

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="4.3. Inlined functions">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="inlined-function"></a>4.3. Inlined functions</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            You may see that a function is credited with a certain number of samples, but

            
            the listing does not add up to the correct total. To pick a real example :

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
                           :internal_sk_buff_alloc_security(struct sk_buff *skb)

            
             353 2.342%    :{ /* internal_sk_buff_alloc_security total: 1882 12.48% */

            
                           :

            
                           :        sk_buff_security_t *sksec;

            
              15 0.0995%   :        int rc = 0;

            
                           :

            
              10 0.06633%  :        sksec = skb-&gt;lsm_security;

            
             468 3.104%    :        if (sksec &amp;&amp; sksec-&gt;magic == DSI_MAGIC) {

            
                           :                goto out;

            
                           :        }

            
                           :

            
                           :        sksec = (sk_buff_security_t *) get_sk_buff_memory(skb);

            
               3 0.0199%   :        if (!sksec) {

            
              38 0.2521%   :                rc = -ENOMEM;

            
                           :                goto out;

            
              10 0.06633%  :        }

            
                           :        memset(sksec, 0, sizeof (sk_buff_security_t));

            
              44 0.2919%   :        sksec-&gt;magic = DSI_MAGIC;

            
              32 0.2123%   :        sksec-&gt;skb = skb;

            
              45 0.2985%   :        sksec-&gt;sid = DSI_SID_NORMAL;

            
              31 0.2056%   :        skb-&gt;lsm_security = sksec;

            
                           :

            
                           :      out:

            
                           :

            
             146 0.9685%   :        return rc;

            
                           :

            
              98 0.6501%   :}

            
            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            Here, the function is credited with 1,882 samples, but the annotations

            
            below do not account for this. This is usually because of inline functions -

            
            the compiler marks such code with debug entries for the inline function

            
            definition, and this is where <span class="command"><strong>opannotate</strong></span> annotates

            
            such samples. In the case above, <code class="function">memset</code> is the most

            
            likely candidate for this problem. Examining the mixed source/assembly

            
            output can help identify such results.

            
            </p>

            
                        <p>

            
            This problem is more visible when there is no source file available, in the

            
            following example it's trivially visible the sums of symbols samples is less

            
            than the number of the samples for this file. The difference must be accounted

            
            to inline functions.

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
            /*

            
             * Total samples for file : "arch/i386/kernel/process.c"

            
             *

            
             *    109  2.4616

            
             */


             /* default_idle total:     84  1.8970 */

            
             /* cpu_idle total:         21  0.4743 */

            
             /* flush_thread total:      1  0.0226 */

            
             /* prepare_to_copy total:   1  0.0226 */

            
             /* __switch_to total:      18  0.4065 */

            
            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            The missing samples are not lost, they will be credited to another source

            
            location where the inlined function is defined. The inlined function will be

            
            credited from multiple call site and merged in one place in the annotated

            
            source file so there is no way to see from what call site are coming the

            
            samples for an inlined function.

            
            </p>

            
                        <p>

            
            When running <span class="command"><strong>opannotate</strong></span>, you may get a warning

            
            "some functions compiled without debug information may have incorrect source line attributions".

            
            In some rare cases, OProfile is not able to verify that the derived source line

            
            is correct (when some parts of the binary image are compiled without debugging

            
            information). Be wary of results if this warning appears.

            
            </p>

            
                        <p>

            
            Furthermore, for some languages the compiler can implicitly generate functions,

            
            such as default copy constructors. Such functions are labelled by the compiler

            
            as having a line number of 0, which means the source annotation can be confusing.

            
            </p>

            
                      </div>

            
                      <div class="sect2" title="4.4. Inaccuracy in line number information">

            
                        <div class="titlepage">

            
                          <div>

            
                            <div>

            
                              <h3 class="title"><a id="wrong-linenr-info"></a>4.4. Inaccuracy in line number information</h3>

            
                            </div>

            
                          </div>

            
                        </div>

            
                        <p>

            
            Depending on your compiler you can fall into the following problem:

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
            struct big_object { int a[500]; };


            int main()

            
            {

            
            	big_object a, b;

            
            	for (int i = 0 ; i != 1000 * 1000; ++i)

            
            		b = a;

            
            	return 0;

            
            }


            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            Compiled with <span class="command"><strong>gcc</strong></span> 3.0.4 the annotated source is clearly inaccurate:

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
                           :int main()

            
                           :{  /* main total: 7871 100% */

            
                           :        big_object a, b;

            
                           :        for (int i = 0 ; i != 1000 * 1000; ++i)

            
                           :                b = a;

            
             7871 100%     :        return 0;

            
                           :}

            
            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            The problem here is distinct from the IRQ latency problem; the debug line number

            
            information is not precise enough; again, looking at output of <span class="command"><strong>opannoatate -as</strong></span> can help.

            
            </p>

            
                        <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                          <tr>

            
                            <td>

            
                              <pre class="screen">

            
                           :int main()

            
                           :{

            
                           :        big_object a, b;

            
                           :        for (int i = 0 ; i != 1000 * 1000; ++i)

            
                           : 80484c0:       push   %ebp

            
                           : 80484c1:       mov    %esp,%ebp

            
                           : 80484c3:       sub    $0xfac,%esp

            
                           : 80484c9:       push   %edi

            
                           : 80484ca:       push   %esi

            
                           : 80484cb:       push   %ebx

            
                           :                b = a;

            
                           : 80484cc:       lea    0xfffff060(%ebp),%edx

            
                           : 80484d2:       lea    0xfffff830(%ebp),%eax

            
                           : 80484d8:       mov    $0xf423f,%ebx

            
                           : 80484dd:       lea    0x0(%esi),%esi

            
                           :        return 0;

            
                3 0.03811% : 80484e0:       mov    %edx,%edi

            
                           : 80484e2:       mov    %eax,%esi

            
                1 0.0127%  : 80484e4:       cld

            
                8 0.1016%  : 80484e5:       mov    $0x1f4,%ecx

            
             7850 99.73%   : 80484ea:       repz movsl %ds:(%esi),%es:(%edi)

            
                9 0.1143%  : 80484ec:       dec    %ebx

            
                           : 80484ed:       jns    80484e0

            
                           : 80484ef:       xor    %eax,%eax

            
                           : 80484f1:       pop    %ebx

            
                           : 80484f2:       pop    %esi

            
                           : 80484f3:       pop    %edi

            
                           : 80484f4:       leave

            
                           : 80484f5:       ret

            
            </pre>

            
                            </td>

            
                          </tr>

            
                        </table>

            
                        <p>

            
            So here it's clear that copying is correctly credited with of all the samples, but the

            
            line number information is misplaced. <span class="command"><strong>objdump -dS</strong></span> exposes the

            
            same problem. Note that maintaining accurate debug information for compilers when optimizing is difficult, so this problem is not suprising.

            
            The problem of debug information

            
            accuracy is also dependent on the binutils version used; some BFD library versions

            
            contain a work-around for known problems of <span class="command"><strong>gcc</strong></span>, some others do not. This is unfortunate but we must live with that,

            
            since profiling is pointless when you disable optimisation (which would give better debugging entries).

            
            </p>

            
                      </div>

            
                    </div>

            
                    <div class="sect1" title="5. Assembly functions">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="symbol-without-debug-info"></a>5. Assembly functions</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            Often the assembler cannot generate debug information automatically.

            
            This means that you cannot get a source report unless

            
            you manually define the neccessary debug information; read your assembler documentation for how you might

            
            do that. The only

            
            debugging info needed currently by OProfile is the line-number/filename-VMA association. When profiling assembly

            
            without debugging info you can always get report for symbols, and optionally for VMA, through <span class="command"><strong>opreport -l</strong></span>

            
            or <span class="command"><strong>opreport -d</strong></span>, but this works only for symbols with the right attributes.

            
            For <span class="command"><strong>gas</strong></span> you can get this by

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            .globl foo

            
            	.type	foo,@function

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            whilst for <span class="command"><strong>nasm</strong></span> you must use

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            	  GLOBAL foo:function		; [1]

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            Note that OProfile does not need the global attribute, only the function attribute.

            
            </p>

            
                    </div>

            
                    <div class="sect1" title="6. Overlapping symbols in JITed code">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="overlapping-symbols"></a>6. Overlapping symbols in JITed code</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            	Some virtual machines (e.g., Java) may re-JIT a method, resulting in previously

            
            	allocated space for a piece of compiled code to be reused. This means that, at one distinct

            
            	code address, multiple symbols/methods may be present during the run time of the application.

            
            	</p>

            
                      <p>

            
            	Since OProfile samples are buffered and don&#8242;t have timing information, there is no way

            
            	to correlate samples with the (possibly) varying address ranges in which the code for a symbol

            
            	may reside.

            
            	An alternative would be flushing the OProfile sampling buffer when we get an unload event,

            
            	but this could result in high overhead.

            
            	</p>

            
                      <p>

            
            	To moderate the problem of overlapping symbols, OProfile tries to select the symbol that was

            
            	present at this address range most of the time. Additionally, other overlapping symbols

            
            	are truncated in the overlapping area.

            
            	This gives reasonable results, because in reality, address reuse typically takes place

            
            	during phase changes of the application -- in particular, during application  startup.

            
            	Thus, for optimum profiling results, start the sampling session after application startup

            
            	and burn in.

            
            	</p>

            
                    </div>

            
                    <div class="sect1" title="7. Using operf to profile fork/execs">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="interpreting_operf_results"></a>7. Using operf to profile fork/execs</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            When profiling an application that forks one or more new processes, <span class="command"><strong>operf</strong></span> will

            
            record samples for both the parent process and forked processes. This is also true even if the

            
            forked process performs an exec of some sort (e.g., <code class="code">execvp</code>). If the

            
            process does <span class="emphasis"><em>not</em></span> perform an exec, you will see that <span class="command"><strong>opreport</strong></span>

            
            will attribute samples for the forked process to the main application executable.  On the other

            
            hand, if the forked process <span class="emphasis"><em>does</em></span> perform an exec, then <span class="command"><strong>opreport</strong></span>

            
            will attribute samples to the executable being exec'ed.

            
            </p>

            
                      <p>

            
            To demonstrate this, consider the following examples.

            
            When using <span class="command"><strong>operf</strong></span> to profile a single application (either with the <code class="code">--pid</code>

            
            option or <code class="code">command</code> option), the normal <span class="command"><strong>opreport</strong></span> summary output

            
            (i.e., invoking <span class="command"><strong>opreport</strong></span> with no options) looks something like the following:

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            CPU_CLK_UNHALT...|

            
              samples|      %|

            
            ------------------

            
               112342 100.000 sprintft

            
            	CPU_CLK_UNHALT...|

            
            	  samples|      %|

            
            	------------------

            
            	   104209 92.7605 libc-2.12.so

            
            	     7273  6.4740 sprintft

            
            	      858  0.7637 no-vmlinux

            
            	        2  0.0018 ld-2.12.so

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            </p>

            
                      <p>

            
            But if you profile an application that does a fork/exec, the <span class="command"><strong>opreport</strong></span> summary output

            
            will show samples for both the main application you profiled, as well as the exec'ed program.

            
            An example is shown below where <code class="code">s-m-fork</code> is the main application being profiled, which

            
            in turn forks a process that does an <code class="code">execvp</code> of the <code class="code">memcpyt</code> program.

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">

            
            CPU_CLK_UNHALT...|

            
              samples|      %|

            
            ------------------

            
               133382 70.5031 memcpyt

            
            	CPU_CLK_UNHALT...|

            
            	  samples|      %|

            
            	------------------

            
            	   123852 92.8551 libc-2.12.so

            
            	     8522  6.3892 memcpyt

            
            	     1007  0.7550 no-vmlinux

            
            	        1 7.5e-04 ld-2.12.so

            
                55804 29.4969 s-m-fork

            
            	CPU_CLK_UNHALT...|

            
            	  samples|      %|

            
            	------------------

            
            	    51801 92.8267 libc-2.12.so

            
            	     3589  6.4314 s-m-fork

            
            	      414  0.7419 no-vmlinux

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            </p>

            
                    </div>

            
                    <div class="sect1" title="8. Other discrepancies">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="hidden-cost"></a>8. Other discrepancies</h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            Another cause of apparent problems is the hidden cost of instructions. A very

            
            common example is two memory reads: one from L1 cache and the other from memory:

            
            the second memory read is likely to have more samples.

            
            There are many other causes of hidden cost of instructions. A non-exhaustive

            
            list: mis-predicted branch, TLB cache miss, partial register stall,

            
            partial register dependencies, memory mismatch stall, re-executed µops. If you want to write

            
            programs at the assembly level, be sure to take a look at the Intel and

            
            AMD documentation at <a class="ulink" href="http://developer.intel.com/">http://developer.intel.com/</a>

            
            and <a class="ulink" href="http://developer.amd.com/devguides.jsp/">http://developer.amd.com/devguides.jsp</a>.

            
            </p>

            
                    </div>

            
                  </div>

            
                  <div class="chapter" title="Chapter 6. Controlling the event counter">

            
                    <div class="titlepage">

            
                      <div>

            
                        <div>

            
                          <h2 class="title"><a id="controlling-counter"></a>Chapter 6. Controlling the event counter</h2>

            
                        </div>

            
                      </div>

            
                    </div>

            
                    <div class="toc">

            
                      <p>

            
                        <b>Table of Contents</b>

            
                      </p>

            
                      <dl>

            
                        <dt>

            
                          <span class="sect1">

            
                            <a href="#controlling-ocount">1. Using <span class="command"><strong>ocount</strong></span></a>

            
                          </span>

            
                        </dt>

            
                      </dl>

            
                    </div>

            
                    <div class="sect1" title="1. Using ocount">

            
                      <div class="titlepage">

            
                        <div>

            
                          <div>

            
                            <h2 class="title" style="clear: both"><a id="controlling-ocount"></a>1. Using <span class="command"><strong>ocount</strong></span></h2>

            
                          </div>

            
                        </div>

            
                      </div>

            
                      <p>

            
            This section describes in detail how <span class="command"><strong>ocount</strong></span> is used.

            
            Unless the <code class="option">--events</code> option is specified, <span class="command"><strong>ocount</strong></span> will use

            
            the default event for your system. For most systems, the default event is some

            
            cycles-based event, assuming your processor type supports hardware performance

            
            counters. The event specification used for <span class="command"><strong>ocount</strong></span> is slightly

            
            different from that required for profiling -- a <span class="emphasis"><em>count</em></span> value

            
            is not needed. You can see the event information for your CPU using <span class="command"><strong>ophelp</strong></span>.

            
            More information on event specification can be found at <a class="xref" href="#eventspec" title="4. Specifying performance counter events">Section 4, &#8220;Specifying performance counter events&#8221;</a>.

            
            </p>

            
                      <p>

            
            The <span class="command"><strong>ocount</strong></span> command syntax is:

            
            </p>

            
                      <p>

            
            </p>

            
                      <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                        <tr>

            
                          <td>

            
                            <pre class="screen">ocount [ options ] [ --system-wide | --process-list &lt;pids&gt; | --thread-list &lt;tids&gt; | --cpu-list &lt;cpus&gt; [ command [ args ] ] ]

            
            </pre>

            
                          </td>

            
                        </tr>

            
                      </table>

            
                      <p>

            
            </p>

            
                      <p>

            
            <span class="command"><strong>ocount</strong></span> has 5 run modes:

            
            </p>

            
                      <p>

            
            </p>

            
                      <div class="itemizedlist">

            
                        <ul class="itemizedlist" type="disc">

            
                          <li class="listitem">system-wide</li>

            
                          <li class="listitem">process-list</li>

            
                          <li class="listitem">thread-list</li>

            
                          <li class="listitem">cpu-list</li>

            
                          <li class="listitem">command</li>

            
                        </ul>

            
                      </div>

            
                      <p>

            
            </p>

            
                      <p>

            
            One and only one of these 5 run modes must be specified when you run <span class="command"><strong>ocount</strong></span>.

            
            If you run <span class="command"><strong>ocount</strong></span> using a run mode other than <code class="code">command [args]</code>, press Ctrl-c

            
            to stop it when finished counting (e.g., when the monitored process ends). If you background <span class="command"><strong>ocount</strong></span>

            
            (i.e., with &#8217;&amp;&#8217;) while using one these run modes, you must stop it in a controlled manner so that

            
            the data collection process can be shut down cleanly and final results can be displayed.

            
            Use <code class="code">kill -SIGINT &lt;ocount-PID&gt;</code> for this purpose.

            
            </p>

            
                      <p>

            
            Following is a description of the <span class="command"><strong>ocount</strong></span> options.

            
            </p>

            
                      <div class="variablelist">

            
                        <dl>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">command [args]</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		The command or application to be profiled. The <span class="emphasis"><em>[args]</em></span> are the input arguments

            
                    that the command or application requires. The command and its arguments must be positioned at the

            
                    end of the command line, after all other <span class="command"><strong>ocount</strong></span> options.

            
                    </p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--process-list / -p [PIDs]</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Use this option to count events for one or more already-running applications, specified via

            
                    a comma-separated list (PIDs). Event counts will be collected for all children of the

            
                    passed process(es) as well.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--thread-list / -r [TIDs]</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Use this option to count events for one or more already-running threads, specified via

            
                    a comma-separated list (TIDs). Event counts will <span class="emphasis"><em>not</em></span> be collected

            
                    for any children of the passed thread(s).

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--system-wide / -s</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This option is for counting events for all processes running on your system. You must have

            
                    root authority to run <span class="command"><strong>ocount</strong></span> in this mode.

            
                    </p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--cpu-list / -C [CPUs]</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This option is for counting events on a subset of processors on your system. You must have

            
                    root authority to run <span class="command"><strong>ocount</strong></span> in this mode. This is a comma-separated list,

            
                    where each element in the list may be either a single processor number or a range of processor

            
                    numbers; for example: &#8217;-C 2,3,4-11,15&#8217;.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--events / -e  [event1[,event2[,...]]]</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This option is for passing a comma-separated list of event specifications

            
            		for counting. Each event spec is of the form:

            
            		</p>

            
                            <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                              <tr>

            
                                <td>

            
                                  <pre class="screen">name[:unitmask[:kernel[:user]]]</pre>

            
                                </td>

            
                              </tr>

            
                            </table>

            
                            <p>

            
            		When no event specification is given, the default event for the running

            
            		processor type will be used for counting. Use <span class="command"><strong>ophelp</strong></span>

            
            		to list the available events for your processor type.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--separate-thread / -t</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
                    This option can be used in conjunction with either the <code class="code">--process-list</code> or

            
                    <code class="code">--thread-list</code> option to display event counts on a per-thread (per-process) basis.

            
                    Without this option, all counts are aggregated.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--separate-cpu / -c</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		This option can be used in conjunction with either the <code class="code">--system-wide</code> or

            
            		<code class="code">--cpu-list</code> option to display event counts on a per-cpu basis. Without this option,

            
            		all counts are aggregated.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--time-interval / -i num_seconds[:num_intervals]</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Results collected for each time interval are printed every num_seconds instead of the

            
                    default of one dump of cumulative event counts at the end of the run. If <code class="code">num_intervals</code>

            
                    is specified, <span class="command"><strong>ocount</strong></span> exits after the specified number of intervals occur.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--brief-format / -b</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Use this option to print results in the following brief format:

            
            		</p>

            
                            <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                              <tr>

            
                                <td>

            
                                  <pre class="screen">

            
                              [optional cpu or thread,]&lt;event_name&gt;,&lt;count&gt;,&lt;percent_time_enabled&gt;

            
                              [        &lt;int&gt;         ,]&lt;  string  &gt;,&lt; u64 &gt;,&lt;     double         &gt;

            
                    </pre>

            
                                </td>

            
                              </tr>

            
                            </table>

            
                            <p>

            
                    If <code class="code">--timer-interval</code> is specified, a separate line formatted as

            
                    </p>

            
                            <table xmlns="" border="0" style="background: #E0E0E0;" width="90%">

            
                              <tr>

            
                                <td>

            
                                  <pre class="screen">

            
                              timestamp,&lt;num_seconds_since_epoch&gt;

            
                    </pre>

            
                                </td>

            
                              </tr>

            
                            </table>

            
                            <p>

            
                    is printed ahead of each dump of event counts.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--output-file / -f outfile_name</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Results are written to outfile_name instead of interactively to the terminal.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--verbose / -V</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Use this option to increase the verbosity of the output.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--version -v </code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Show <span class="command"><strong>ocount</strong></span> version.

            
            		</p>

            
                          </dd>

            
                          <dt>

            
                            <span class="term">

            
                              <code class="option">--help / -h</code>

            
                            </span>

            
                          </dt>

            
                          <dd>

            
                            <p>

            
            		Show a help message.

            
            		</p>

            
                          </dd>

            
                        </dl>

            
                      </div>

            
                    </div>

            
                  </div>

            
                  <div class="chapter" title="Chapter 7. Acknowledgments">

            
                    <div class="titlepage">

            
                      <div>

            
                        <div>

            
                          <h2 class="title"><a id="ack"></a>Chapter 7. Acknowledgments</h2>

            
                        </div>

            
                      </div>

            
                    </div>

            
                    <p>

            
            Thanks to (in no particular order) : Arjan van de Ven, Rik van Riel, Juan Quintela, Philippe Elie,

            
            Phillipp Rumpf, Tigran Aivazian, Alex Brown, Alisdair Rawsthorne, Bob Montgomery, Ray Bryant, H.J. Lu,

            
            Jeff Esper, Will Cohen, Graydon Hoare, Cliff Woolley, Alex Tsariounov, Al Stone, Jason Yeh,

            
            Randolph Chung, Anton Blanchard, Richard Henderson, Andries Brouwer, Bryan Rittmeyer,

            
            Maynard P. Johnson,

            
            Richard Reich (rreich@rdrtech.com), Zwane Mwaikambo, Dave Jones, Charles Filtness; and finally Pulp, for "Intro".

            
            </p>

            
                  </div>

            
                </div>

            
              </body>

            
            </html>