| /* |
| * |
| * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. |
| * |
| * This program is free software and is provided to you under the terms of the |
| * GNU General Public License version 2 as published by the Free Software |
| * Foundation, and any use by you of this program is subject to the terms |
| * of such GNU licence. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| * You should have received a copy of the GNU General Public License |
| * along with this program; if not, you can access it online at |
| * http://www.gnu.org/licenses/gpl-2.0.html. |
| * |
| * SPDX-License-Identifier: GPL-2.0 |
| * |
| */ |
| |
| #include "mali_kbase_hwcnt_virtualizer.h" |
| #include "mali_kbase_hwcnt_accumulator.h" |
| #include "mali_kbase_hwcnt_context.h" |
| #include "mali_kbase_hwcnt_types.h" |
| #include "mali_malisw.h" |
| #include "mali_kbase_debug.h" |
| #include "mali_kbase_linux.h" |
| |
| #include <linux/mutex.h> |
| #include <linux/slab.h> |
| |
| /** |
| * struct kbase_hwcnt_virtualizer - Hardware counter virtualizer structure. |
| * @hctx: Hardware counter context being virtualized. |
| * @dump_threshold_ns: Minimum threshold period for dumps between different |
| * clients where a new accumulator dump will not be |
| * performed, and instead accumulated values will be used. |
| * If 0, rate limiting is disabled. |
| * @metadata: Hardware counter metadata. |
| * @lock: Lock acquired at all entrypoints, to protect mutable |
| * state. |
| * @client_count: Current number of virtualizer clients. |
| * @clients: List of virtualizer clients. |
| * @accum: Hardware counter accumulator. NULL if no clients. |
| * @scratch_map: Enable map used as scratch space during counter changes. |
| * @scratch_buf: Dump buffer used as scratch space during dumps. |
| * @ts_last_dump_ns: End time of most recent dump across all clients. |
| */ |
| struct kbase_hwcnt_virtualizer { |
| struct kbase_hwcnt_context *hctx; |
| u64 dump_threshold_ns; |
| const struct kbase_hwcnt_metadata *metadata; |
| struct mutex lock; |
| size_t client_count; |
| struct list_head clients; |
| struct kbase_hwcnt_accumulator *accum; |
| struct kbase_hwcnt_enable_map scratch_map; |
| struct kbase_hwcnt_dump_buffer scratch_buf; |
| u64 ts_last_dump_ns; |
| }; |
| |
| /** |
| * struct kbase_hwcnt_virtualizer_client - Virtualizer client structure. |
| * @node: List node used for virtualizer client list. |
| * @hvirt: Hardware counter virtualizer. |
| * @enable_map: Enable map with client's current enabled counters. |
| * @accum_buf: Dump buffer with client's current accumulated counters. |
| * @has_accum: True if accum_buf contains any accumulated counters. |
| * @ts_start_ns: Counter collection start time of current dump. |
| */ |
| struct kbase_hwcnt_virtualizer_client { |
| struct list_head node; |
| struct kbase_hwcnt_virtualizer *hvirt; |
| struct kbase_hwcnt_enable_map enable_map; |
| struct kbase_hwcnt_dump_buffer accum_buf; |
| bool has_accum; |
| u64 ts_start_ns; |
| }; |
| |
| const struct kbase_hwcnt_metadata *kbase_hwcnt_virtualizer_metadata( |
| struct kbase_hwcnt_virtualizer *hvirt) |
| { |
| if (!hvirt) |
| return NULL; |
| |
| return hvirt->metadata; |
| } |
| KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_metadata); |
| |
| /** |
| * kbasep_hwcnt_virtualizer_client_free - Free a virtualizer client's memory. |
| * @hvcli: Pointer to virtualizer client. |
| * |
| * Will safely free a client in any partial state of construction. |
| */ |
| static void kbasep_hwcnt_virtualizer_client_free( |
| struct kbase_hwcnt_virtualizer_client *hvcli) |
| { |
| if (!hvcli) |
| return; |
| |
| kbase_hwcnt_dump_buffer_free(&hvcli->accum_buf); |
| kbase_hwcnt_enable_map_free(&hvcli->enable_map); |
| kfree(hvcli); |
| } |
| |
| /** |
| * kbasep_hwcnt_virtualizer_client_alloc - Allocate memory for a virtualizer |
| * client. |
| * @metadata: Non-NULL pointer to counter metadata. |
| * @out_hvcli: Non-NULL pointer to where created client will be stored on |
| * success. |
| * |
| * Return: 0 on success, else error code. |
| */ |
| static int kbasep_hwcnt_virtualizer_client_alloc( |
| const struct kbase_hwcnt_metadata *metadata, |
| struct kbase_hwcnt_virtualizer_client **out_hvcli) |
| { |
| int errcode; |
| struct kbase_hwcnt_virtualizer_client *hvcli = NULL; |
| |
| WARN_ON(!metadata); |
| WARN_ON(!out_hvcli); |
| |
| hvcli = kzalloc(sizeof(*hvcli), GFP_KERNEL); |
| if (!hvcli) |
| return -ENOMEM; |
| |
| errcode = kbase_hwcnt_enable_map_alloc(metadata, &hvcli->enable_map); |
| if (errcode) |
| goto error; |
| |
| errcode = kbase_hwcnt_dump_buffer_alloc(metadata, &hvcli->accum_buf); |
| if (errcode) |
| goto error; |
| |
| *out_hvcli = hvcli; |
| return 0; |
| error: |
| kbasep_hwcnt_virtualizer_client_free(hvcli); |
| return errcode; |
| } |
| |
| /** |
| * kbasep_hwcnt_virtualizer_client_accumulate - Accumulate a dump buffer into a |
| * client's accumulation buffer. |
| * @hvcli: Non-NULL pointer to virtualizer client. |
| * @dump_buf: Non-NULL pointer to dump buffer to accumulate from. |
| */ |
| static void kbasep_hwcnt_virtualizer_client_accumulate( |
| struct kbase_hwcnt_virtualizer_client *hvcli, |
| const struct kbase_hwcnt_dump_buffer *dump_buf) |
| { |
| WARN_ON(!hvcli); |
| WARN_ON(!dump_buf); |
| lockdep_assert_held(&hvcli->hvirt->lock); |
| |
| if (hvcli->has_accum) { |
| /* If already some accumulation, accumulate */ |
| kbase_hwcnt_dump_buffer_accumulate( |
| &hvcli->accum_buf, dump_buf, &hvcli->enable_map); |
| } else { |
| /* If no accumulation, copy */ |
| kbase_hwcnt_dump_buffer_copy( |
| &hvcli->accum_buf, dump_buf, &hvcli->enable_map); |
| } |
| hvcli->has_accum = true; |
| } |
| |
| /** |
| * kbasep_hwcnt_virtualizer_accumulator_term - Terminate the hardware counter |
| * accumulator after final client |
| * removal. |
| * @hvirt: Non-NULL pointer to the hardware counter virtualizer. |
| * |
| * Will safely terminate the accumulator in any partial state of initialisation. |
| */ |
| static void kbasep_hwcnt_virtualizer_accumulator_term( |
| struct kbase_hwcnt_virtualizer *hvirt) |
| { |
| WARN_ON(!hvirt); |
| lockdep_assert_held(&hvirt->lock); |
| WARN_ON(hvirt->client_count); |
| |
| kbase_hwcnt_dump_buffer_free(&hvirt->scratch_buf); |
| kbase_hwcnt_enable_map_free(&hvirt->scratch_map); |
| kbase_hwcnt_accumulator_release(hvirt->accum); |
| hvirt->accum = NULL; |
| } |
| |
| /** |
| * kbasep_hwcnt_virtualizer_accumulator_init - Initialise the hardware counter |
| * accumulator before first client |
| * addition. |
| * @hvirt: Non-NULL pointer to the hardware counter virtualizer. |
| * |
| * Return: 0 on success, else error code. |
| */ |
| static int kbasep_hwcnt_virtualizer_accumulator_init( |
| struct kbase_hwcnt_virtualizer *hvirt) |
| { |
| int errcode; |
| |
| WARN_ON(!hvirt); |
| lockdep_assert_held(&hvirt->lock); |
| WARN_ON(hvirt->client_count); |
| WARN_ON(hvirt->accum); |
| |
| errcode = kbase_hwcnt_accumulator_acquire( |
| hvirt->hctx, &hvirt->accum); |
| if (errcode) |
| goto error; |
| |
| errcode = kbase_hwcnt_enable_map_alloc( |
| hvirt->metadata, &hvirt->scratch_map); |
| if (errcode) |
| goto error; |
| |
| errcode = kbase_hwcnt_dump_buffer_alloc( |
| hvirt->metadata, &hvirt->scratch_buf); |
| if (errcode) |
| goto error; |
| |
| return 0; |
| error: |
| kbasep_hwcnt_virtualizer_accumulator_term(hvirt); |
| return errcode; |
| } |
| |
| /** |
| * kbasep_hwcnt_virtualizer_client_add - Add a newly allocated client to the |
| * virtualizer. |
| * @hvirt: Non-NULL pointer to the hardware counter virtualizer. |
| * @hvcli: Non-NULL pointer to the virtualizer client to add. |
| * @enable_map: Non-NULL pointer to client's initial enable map. |
| * |
| * Return: 0 on success, else error code. |
| */ |
| static int kbasep_hwcnt_virtualizer_client_add( |
| struct kbase_hwcnt_virtualizer *hvirt, |
| struct kbase_hwcnt_virtualizer_client *hvcli, |
| const struct kbase_hwcnt_enable_map *enable_map) |
| { |
| int errcode = 0; |
| u64 ts_start_ns; |
| u64 ts_end_ns; |
| |
| WARN_ON(!hvirt); |
| WARN_ON(!hvcli); |
| WARN_ON(!enable_map); |
| lockdep_assert_held(&hvirt->lock); |
| |
| if (hvirt->client_count == 0) |
| /* First client added, so initialise the accumulator */ |
| errcode = kbasep_hwcnt_virtualizer_accumulator_init(hvirt); |
| if (errcode) |
| return errcode; |
| |
| hvirt->client_count += 1; |
| |
| if (hvirt->client_count == 1) { |
| /* First client, so just pass the enable map onwards as is */ |
| errcode = kbase_hwcnt_accumulator_set_counters(hvirt->accum, |
| enable_map, &ts_start_ns, &ts_end_ns, NULL); |
| } else { |
| struct kbase_hwcnt_virtualizer_client *pos; |
| |
| /* Make the scratch enable map the union of all enable maps */ |
| kbase_hwcnt_enable_map_copy( |
| &hvirt->scratch_map, enable_map); |
| list_for_each_entry(pos, &hvirt->clients, node) |
| kbase_hwcnt_enable_map_union( |
| &hvirt->scratch_map, &pos->enable_map); |
| |
| /* Set the counters with the new union enable map */ |
| errcode = kbase_hwcnt_accumulator_set_counters(hvirt->accum, |
| &hvirt->scratch_map, |
| &ts_start_ns, &ts_end_ns, |
| &hvirt->scratch_buf); |
| /* Accumulate into only existing clients' accumulation bufs */ |
| if (!errcode) |
| list_for_each_entry(pos, &hvirt->clients, node) |
| kbasep_hwcnt_virtualizer_client_accumulate( |
| pos, &hvirt->scratch_buf); |
| } |
| if (errcode) |
| goto error; |
| |
| list_add(&hvcli->node, &hvirt->clients); |
| hvcli->hvirt = hvirt; |
| kbase_hwcnt_enable_map_copy(&hvcli->enable_map, enable_map); |
| hvcli->has_accum = false; |
| hvcli->ts_start_ns = ts_end_ns; |
| |
| /* Store the most recent dump time for rate limiting */ |
| hvirt->ts_last_dump_ns = ts_end_ns; |
| |
| return 0; |
| error: |
| hvirt->client_count -= 1; |
| if (hvirt->client_count == 0) |
| kbasep_hwcnt_virtualizer_accumulator_term(hvirt); |
| return errcode; |
| } |
| |
| /** |
| * kbasep_hwcnt_virtualizer_client_remove - Remove a client from the |
| * virtualizer. |
| * @hvirt: Non-NULL pointer to the hardware counter virtualizer. |
| * @hvcli: Non-NULL pointer to the virtualizer client to remove. |
| */ |
| static void kbasep_hwcnt_virtualizer_client_remove( |
| struct kbase_hwcnt_virtualizer *hvirt, |
| struct kbase_hwcnt_virtualizer_client *hvcli) |
| { |
| int errcode = 0; |
| u64 ts_start_ns; |
| u64 ts_end_ns; |
| |
| WARN_ON(!hvirt); |
| WARN_ON(!hvcli); |
| lockdep_assert_held(&hvirt->lock); |
| |
| list_del(&hvcli->node); |
| hvirt->client_count -= 1; |
| |
| if (hvirt->client_count == 0) { |
| /* Last client removed, so terminate the accumulator */ |
| kbasep_hwcnt_virtualizer_accumulator_term(hvirt); |
| } else { |
| struct kbase_hwcnt_virtualizer_client *pos; |
| /* Make the scratch enable map the union of all enable maps */ |
| kbase_hwcnt_enable_map_disable_all(&hvirt->scratch_map); |
| list_for_each_entry(pos, &hvirt->clients, node) |
| kbase_hwcnt_enable_map_union( |
| &hvirt->scratch_map, &pos->enable_map); |
| /* Set the counters with the new union enable map */ |
| errcode = kbase_hwcnt_accumulator_set_counters(hvirt->accum, |
| &hvirt->scratch_map, |
| &ts_start_ns, &ts_end_ns, |
| &hvirt->scratch_buf); |
| /* Accumulate into remaining clients' accumulation bufs */ |
| if (!errcode) |
| list_for_each_entry(pos, &hvirt->clients, node) |
| kbasep_hwcnt_virtualizer_client_accumulate( |
| pos, &hvirt->scratch_buf); |
| |
| /* Store the most recent dump time for rate limiting */ |
| hvirt->ts_last_dump_ns = ts_end_ns; |
| } |
| WARN_ON(errcode); |
| } |
| |
| /** |
| * kbasep_hwcnt_virtualizer_client_set_counters - Perform a dump of the client's |
| * currently enabled counters, |
| * and enable a new set of |
| * counters that will be used for |
| * subsequent dumps. |
| * @hvirt: Non-NULL pointer to the hardware counter virtualizer. |
| * @hvcli: Non-NULL pointer to the virtualizer client. |
| * @enable_map: Non-NULL pointer to the new counter enable map for the client. |
| * Must have the same metadata as the virtualizer. |
| * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will |
| * be written out to on success. |
| * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will |
| * be written out to on success. |
| * @dump_buf: Pointer to the buffer where the dump will be written out to on |
| * success. If non-NULL, must have the same metadata as the |
| * accumulator. If NULL, the dump will be discarded. |
| * |
| * Return: 0 on success or error code. |
| */ |
| static int kbasep_hwcnt_virtualizer_client_set_counters( |
| struct kbase_hwcnt_virtualizer *hvirt, |
| struct kbase_hwcnt_virtualizer_client *hvcli, |
| const struct kbase_hwcnt_enable_map *enable_map, |
| u64 *ts_start_ns, |
| u64 *ts_end_ns, |
| struct kbase_hwcnt_dump_buffer *dump_buf) |
| { |
| int errcode; |
| struct kbase_hwcnt_virtualizer_client *pos; |
| |
| WARN_ON(!hvirt); |
| WARN_ON(!hvcli); |
| WARN_ON(!enable_map); |
| WARN_ON(!ts_start_ns); |
| WARN_ON(!ts_end_ns); |
| WARN_ON(enable_map->metadata != hvirt->metadata); |
| WARN_ON(dump_buf && (dump_buf->metadata != hvirt->metadata)); |
| lockdep_assert_held(&hvirt->lock); |
| |
| /* Make the scratch enable map the union of all enable maps */ |
| kbase_hwcnt_enable_map_copy(&hvirt->scratch_map, enable_map); |
| list_for_each_entry(pos, &hvirt->clients, node) |
| /* Ignore the enable map of the selected client */ |
| if (pos != hvcli) |
| kbase_hwcnt_enable_map_union( |
| &hvirt->scratch_map, &pos->enable_map); |
| |
| /* Set the counters with the new union enable map */ |
| errcode = kbase_hwcnt_accumulator_set_counters(hvirt->accum, |
| &hvirt->scratch_map, ts_start_ns, ts_end_ns, |
| &hvirt->scratch_buf); |
| if (errcode) |
| return errcode; |
| |
| /* Accumulate into all accumulation bufs except the selected client's */ |
| list_for_each_entry(pos, &hvirt->clients, node) |
| if (pos != hvcli) |
| kbasep_hwcnt_virtualizer_client_accumulate( |
| pos, &hvirt->scratch_buf); |
| |
| /* Finally, write into the dump buf */ |
| if (dump_buf) { |
| const struct kbase_hwcnt_dump_buffer *src = &hvirt->scratch_buf; |
| |
| if (hvcli->has_accum) { |
| kbase_hwcnt_dump_buffer_accumulate( |
| &hvcli->accum_buf, src, &hvcli->enable_map); |
| src = &hvcli->accum_buf; |
| } |
| kbase_hwcnt_dump_buffer_copy(dump_buf, src, &hvcli->enable_map); |
| } |
| hvcli->has_accum = false; |
| |
| /* Update the selected client's enable map */ |
| kbase_hwcnt_enable_map_copy(&hvcli->enable_map, enable_map); |
| |
| /* Fix up the timestamps */ |
| *ts_start_ns = hvcli->ts_start_ns; |
| hvcli->ts_start_ns = *ts_end_ns; |
| |
| /* Store the most recent dump time for rate limiting */ |
| hvirt->ts_last_dump_ns = *ts_end_ns; |
| |
| return errcode; |
| } |
| |
| int kbase_hwcnt_virtualizer_client_set_counters( |
| struct kbase_hwcnt_virtualizer_client *hvcli, |
| const struct kbase_hwcnt_enable_map *enable_map, |
| u64 *ts_start_ns, |
| u64 *ts_end_ns, |
| struct kbase_hwcnt_dump_buffer *dump_buf) |
| { |
| int errcode; |
| struct kbase_hwcnt_virtualizer *hvirt; |
| |
| if (!hvcli || !enable_map || !ts_start_ns || !ts_end_ns) |
| return -EINVAL; |
| |
| hvirt = hvcli->hvirt; |
| |
| if ((enable_map->metadata != hvirt->metadata) || |
| (dump_buf && (dump_buf->metadata != hvirt->metadata))) |
| return -EINVAL; |
| |
| mutex_lock(&hvirt->lock); |
| |
| if ((hvirt->client_count == 1) && (!hvcli->has_accum)) { |
| /* |
| * If there's only one client with no prior accumulation, we can |
| * completely skip the virtualize and just pass through the call |
| * to the accumulator, saving a fair few copies and |
| * accumulations. |
| */ |
| errcode = kbase_hwcnt_accumulator_set_counters( |
| hvirt->accum, enable_map, |
| ts_start_ns, ts_end_ns, dump_buf); |
| |
| if (!errcode) { |
| /* Update the selected client's enable map */ |
| kbase_hwcnt_enable_map_copy( |
| &hvcli->enable_map, enable_map); |
| |
| /* Fix up the timestamps */ |
| *ts_start_ns = hvcli->ts_start_ns; |
| hvcli->ts_start_ns = *ts_end_ns; |
| |
| /* Store the most recent dump time for rate limiting */ |
| hvirt->ts_last_dump_ns = *ts_end_ns; |
| } |
| } else { |
| /* Otherwise, do the full virtualize */ |
| errcode = kbasep_hwcnt_virtualizer_client_set_counters( |
| hvirt, hvcli, enable_map, |
| ts_start_ns, ts_end_ns, dump_buf); |
| } |
| |
| mutex_unlock(&hvirt->lock); |
| |
| return errcode; |
| } |
| KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_set_counters); |
| |
| /** |
| * kbasep_hwcnt_virtualizer_client_dump - Perform a dump of the client's |
| * currently enabled counters. |
| * @hvirt: Non-NULL pointer to the hardware counter virtualizer. |
| * @hvcli: Non-NULL pointer to the virtualizer client. |
| * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will |
| * be written out to on success. |
| * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will |
| * be written out to on success. |
| * @dump_buf: Pointer to the buffer where the dump will be written out to on |
| * success. If non-NULL, must have the same metadata as the |
| * accumulator. If NULL, the dump will be discarded. |
| * |
| * Return: 0 on success or error code. |
| */ |
| static int kbasep_hwcnt_virtualizer_client_dump( |
| struct kbase_hwcnt_virtualizer *hvirt, |
| struct kbase_hwcnt_virtualizer_client *hvcli, |
| u64 *ts_start_ns, |
| u64 *ts_end_ns, |
| struct kbase_hwcnt_dump_buffer *dump_buf) |
| { |
| int errcode; |
| struct kbase_hwcnt_virtualizer_client *pos; |
| |
| WARN_ON(!hvirt); |
| WARN_ON(!hvcli); |
| WARN_ON(!ts_start_ns); |
| WARN_ON(!ts_end_ns); |
| WARN_ON(dump_buf && (dump_buf->metadata != hvirt->metadata)); |
| lockdep_assert_held(&hvirt->lock); |
| |
| /* Perform the dump */ |
| errcode = kbase_hwcnt_accumulator_dump(hvirt->accum, |
| ts_start_ns, ts_end_ns, &hvirt->scratch_buf); |
| if (errcode) |
| return errcode; |
| |
| /* Accumulate into all accumulation bufs except the selected client's */ |
| list_for_each_entry(pos, &hvirt->clients, node) |
| if (pos != hvcli) |
| kbasep_hwcnt_virtualizer_client_accumulate( |
| pos, &hvirt->scratch_buf); |
| |
| /* Finally, write into the dump buf */ |
| if (dump_buf) { |
| const struct kbase_hwcnt_dump_buffer *src = &hvirt->scratch_buf; |
| |
| if (hvcli->has_accum) { |
| kbase_hwcnt_dump_buffer_accumulate( |
| &hvcli->accum_buf, src, &hvcli->enable_map); |
| src = &hvcli->accum_buf; |
| } |
| kbase_hwcnt_dump_buffer_copy(dump_buf, src, &hvcli->enable_map); |
| } |
| hvcli->has_accum = false; |
| |
| /* Fix up the timestamps */ |
| *ts_start_ns = hvcli->ts_start_ns; |
| hvcli->ts_start_ns = *ts_end_ns; |
| |
| /* Store the most recent dump time for rate limiting */ |
| hvirt->ts_last_dump_ns = *ts_end_ns; |
| |
| return errcode; |
| } |
| |
| /** |
| * kbasep_hwcnt_virtualizer_client_dump_rate_limited - Perform a dump of the |
| * client's currently enabled counters |
| * if it hasn't been rate limited, |
| * otherwise return the client's most |
| * recent accumulation. |
| * @hvirt: Non-NULL pointer to the hardware counter virtualizer. |
| * @hvcli: Non-NULL pointer to the virtualizer client. |
| * @ts_start_ns: Non-NULL pointer where the start timestamp of the dump will |
| * be written out to on success. |
| * @ts_end_ns: Non-NULL pointer where the end timestamp of the dump will |
| * be written out to on success. |
| * @dump_buf: Pointer to the buffer where the dump will be written out to on |
| * success. If non-NULL, must have the same metadata as the |
| * accumulator. If NULL, the dump will be discarded. |
| * |
| * Return: 0 on success or error code. |
| */ |
| static int kbasep_hwcnt_virtualizer_client_dump_rate_limited( |
| struct kbase_hwcnt_virtualizer *hvirt, |
| struct kbase_hwcnt_virtualizer_client *hvcli, |
| u64 *ts_start_ns, |
| u64 *ts_end_ns, |
| struct kbase_hwcnt_dump_buffer *dump_buf) |
| { |
| bool rate_limited = true; |
| |
| WARN_ON(!hvirt); |
| WARN_ON(!hvcli); |
| WARN_ON(!ts_start_ns); |
| WARN_ON(!ts_end_ns); |
| WARN_ON(dump_buf && (dump_buf->metadata != hvirt->metadata)); |
| lockdep_assert_held(&hvirt->lock); |
| |
| if (hvirt->dump_threshold_ns == 0) { |
| /* Threshold == 0, so rate limiting disabled */ |
| rate_limited = false; |
| } else if (hvirt->ts_last_dump_ns == hvcli->ts_start_ns) { |
| /* Last dump was performed by this client, and dumps from an |
| * individual client are never rate limited |
| */ |
| rate_limited = false; |
| } else { |
| const u64 ts_ns = |
| kbase_hwcnt_accumulator_timestamp_ns(hvirt->accum); |
| const u64 time_since_last_dump_ns = |
| ts_ns - hvirt->ts_last_dump_ns; |
| |
| /* Dump period equals or exceeds the threshold */ |
| if (time_since_last_dump_ns >= hvirt->dump_threshold_ns) |
| rate_limited = false; |
| } |
| |
| if (!rate_limited) |
| return kbasep_hwcnt_virtualizer_client_dump( |
| hvirt, hvcli, ts_start_ns, ts_end_ns, dump_buf); |
| |
| /* If we've gotten this far, the client must have something accumulated |
| * otherwise it is a logic error |
| */ |
| WARN_ON(!hvcli->has_accum); |
| |
| if (dump_buf) |
| kbase_hwcnt_dump_buffer_copy( |
| dump_buf, &hvcli->accum_buf, &hvcli->enable_map); |
| hvcli->has_accum = false; |
| |
| *ts_start_ns = hvcli->ts_start_ns; |
| *ts_end_ns = hvirt->ts_last_dump_ns; |
| hvcli->ts_start_ns = hvirt->ts_last_dump_ns; |
| |
| return 0; |
| } |
| |
| int kbase_hwcnt_virtualizer_client_dump( |
| struct kbase_hwcnt_virtualizer_client *hvcli, |
| u64 *ts_start_ns, |
| u64 *ts_end_ns, |
| struct kbase_hwcnt_dump_buffer *dump_buf) |
| { |
| int errcode; |
| struct kbase_hwcnt_virtualizer *hvirt; |
| |
| if (!hvcli || !ts_start_ns || !ts_end_ns) |
| return -EINVAL; |
| |
| hvirt = hvcli->hvirt; |
| |
| if (dump_buf && (dump_buf->metadata != hvirt->metadata)) |
| return -EINVAL; |
| |
| mutex_lock(&hvirt->lock); |
| |
| if ((hvirt->client_count == 1) && (!hvcli->has_accum)) { |
| /* |
| * If there's only one client with no prior accumulation, we can |
| * completely skip the virtualize and just pass through the call |
| * to the accumulator, saving a fair few copies and |
| * accumulations. |
| */ |
| errcode = kbase_hwcnt_accumulator_dump( |
| hvirt->accum, ts_start_ns, ts_end_ns, dump_buf); |
| |
| if (!errcode) { |
| /* Fix up the timestamps */ |
| *ts_start_ns = hvcli->ts_start_ns; |
| hvcli->ts_start_ns = *ts_end_ns; |
| |
| /* Store the most recent dump time for rate limiting */ |
| hvirt->ts_last_dump_ns = *ts_end_ns; |
| } |
| } else { |
| /* Otherwise, do the full virtualize */ |
| errcode = kbasep_hwcnt_virtualizer_client_dump_rate_limited( |
| hvirt, hvcli, ts_start_ns, ts_end_ns, dump_buf); |
| } |
| |
| mutex_unlock(&hvirt->lock); |
| |
| return errcode; |
| } |
| KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_dump); |
| |
| int kbase_hwcnt_virtualizer_client_create( |
| struct kbase_hwcnt_virtualizer *hvirt, |
| const struct kbase_hwcnt_enable_map *enable_map, |
| struct kbase_hwcnt_virtualizer_client **out_hvcli) |
| { |
| int errcode; |
| struct kbase_hwcnt_virtualizer_client *hvcli; |
| |
| if (!hvirt || !enable_map || !out_hvcli || |
| (enable_map->metadata != hvirt->metadata)) |
| return -EINVAL; |
| |
| errcode = kbasep_hwcnt_virtualizer_client_alloc( |
| hvirt->metadata, &hvcli); |
| if (errcode) |
| return errcode; |
| |
| mutex_lock(&hvirt->lock); |
| |
| errcode = kbasep_hwcnt_virtualizer_client_add(hvirt, hvcli, enable_map); |
| |
| mutex_unlock(&hvirt->lock); |
| |
| if (errcode) { |
| kbasep_hwcnt_virtualizer_client_free(hvcli); |
| return errcode; |
| } |
| |
| *out_hvcli = hvcli; |
| return 0; |
| } |
| KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_create); |
| |
| void kbase_hwcnt_virtualizer_client_destroy( |
| struct kbase_hwcnt_virtualizer_client *hvcli) |
| { |
| if (!hvcli) |
| return; |
| |
| mutex_lock(&hvcli->hvirt->lock); |
| |
| kbasep_hwcnt_virtualizer_client_remove(hvcli->hvirt, hvcli); |
| |
| mutex_unlock(&hvcli->hvirt->lock); |
| |
| kbasep_hwcnt_virtualizer_client_free(hvcli); |
| } |
| KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_destroy); |
| |
| int kbase_hwcnt_virtualizer_init( |
| struct kbase_hwcnt_context *hctx, |
| u64 dump_threshold_ns, |
| struct kbase_hwcnt_virtualizer **out_hvirt) |
| { |
| struct kbase_hwcnt_virtualizer *virt; |
| const struct kbase_hwcnt_metadata *metadata; |
| |
| if (!hctx || !out_hvirt) |
| return -EINVAL; |
| |
| metadata = kbase_hwcnt_context_metadata(hctx); |
| if (!metadata) |
| return -EINVAL; |
| |
| virt = kzalloc(sizeof(*virt), GFP_KERNEL); |
| if (!virt) |
| return -ENOMEM; |
| |
| virt->hctx = hctx; |
| virt->dump_threshold_ns = dump_threshold_ns; |
| virt->metadata = metadata; |
| |
| mutex_init(&virt->lock); |
| INIT_LIST_HEAD(&virt->clients); |
| |
| *out_hvirt = virt; |
| return 0; |
| } |
| KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_init); |
| |
| void kbase_hwcnt_virtualizer_term( |
| struct kbase_hwcnt_virtualizer *hvirt) |
| { |
| if (!hvirt) |
| return; |
| |
| /* Non-zero client count implies client leak */ |
| if (WARN_ON(hvirt->client_count != 0)) { |
| struct kbase_hwcnt_virtualizer_client *pos, *n; |
| |
| list_for_each_entry_safe(pos, n, &hvirt->clients, node) |
| kbase_hwcnt_virtualizer_client_destroy(pos); |
| } |
| |
| WARN_ON(hvirt->client_count != 0); |
| WARN_ON(hvirt->accum); |
| |
| kfree(hvirt); |
| } |
| KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_term); |
