| // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note |
| /* |
| * |
| * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. |
| * |
| * This program is free software and is provided to you under the terms of the |
| * GNU General Public License version 2 as published by the Free Software |
| * Foundation, and any use by you of this program is subject to the terms |
| * of such GNU license. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| * You should have received a copy of the GNU General Public License |
| * along with this program; if not, you can access it online at |
| * http://www.gnu.org/licenses/gpl-2.0.html. |
| * |
| */ |
| |
| #include "mali_kbase_csf_tl_reader.h" |
| |
| #include "mali_kbase_csf_trace_buffer.h" |
| #include "mali_kbase_reset_gpu.h" |
| |
| #include "tl/mali_kbase_tlstream.h" |
| #include "tl/mali_kbase_tl_serialize.h" |
| #include "tl/mali_kbase_tracepoints.h" |
| |
| #include "mali_kbase_pm.h" |
| #include "mali_kbase_hwaccess_time.h" |
| |
| #include <linux/gcd.h> |
| #include <linux/math64.h> |
| #include <asm/arch_timer.h> |
| |
| #if IS_ENABLED(CONFIG_DEBUG_FS) |
| #include "tl/mali_kbase_timeline_priv.h" |
| #include <linux/debugfs.h> |
| |
| #if (KERNEL_VERSION(4, 7, 0) > LINUX_VERSION_CODE) |
| #define DEFINE_DEBUGFS_ATTRIBUTE DEFINE_SIMPLE_ATTRIBUTE |
| #endif |
| #endif |
| |
| /* Name of the CSFFW timeline tracebuffer. */ |
| #define KBASE_CSFFW_TRACEBUFFER_NAME "timeline" |
| /* Name of the timeline header metatadata */ |
| #define KBASE_CSFFW_TIMELINE_HEADER_NAME "timeline_header" |
| |
| /** |
| * struct kbase_csffw_tl_message - CSFFW timeline message. |
| * |
| * @msg_id: Message ID. |
| * @timestamp: Timestamp of the event. |
| * @cycle_counter: Cycle number of the event. |
| * |
| * Contain fields that are common for all CSFFW timeline messages. |
| */ |
| struct kbase_csffw_tl_message { |
| u32 msg_id; |
| u64 timestamp; |
| u64 cycle_counter; |
| } __packed __aligned(4); |
| |
| #if IS_ENABLED(CONFIG_DEBUG_FS) |
| static int kbase_csf_tl_debugfs_poll_interval_read(void *data, u64 *val) |
| { |
| struct kbase_device *kbdev = (struct kbase_device *)data; |
| struct kbase_csf_tl_reader *self = &kbdev->timeline->csf_tl_reader; |
| |
| *val = self->timer_interval; |
| |
| return 0; |
| } |
| |
| static int kbase_csf_tl_debugfs_poll_interval_write(void *data, u64 val) |
| { |
| struct kbase_device *kbdev = (struct kbase_device *)data; |
| struct kbase_csf_tl_reader *self = &kbdev->timeline->csf_tl_reader; |
| |
| if (val > KBASE_CSF_TL_READ_INTERVAL_MAX || val < KBASE_CSF_TL_READ_INTERVAL_MIN) { |
| return -EINVAL; |
| } |
| |
| self->timer_interval = (u32)val; |
| |
| return 0; |
| } |
| |
| DEFINE_DEBUGFS_ATTRIBUTE(kbase_csf_tl_poll_interval_fops, |
| kbase_csf_tl_debugfs_poll_interval_read, |
| kbase_csf_tl_debugfs_poll_interval_write, "%llu\n"); |
| |
| |
| void kbase_csf_tl_reader_debugfs_init(struct kbase_device *kbdev) |
| { |
| debugfs_create_file("csf_tl_poll_interval_in_ms", S_IRUGO | S_IWUSR, |
| kbdev->debugfs_instr_directory, kbdev, |
| &kbase_csf_tl_poll_interval_fops); |
| |
| } |
| #endif |
| |
| /** |
| * get_cpu_gpu_time() - Get current CPU and GPU timestamps. |
| * |
| * @kbdev: Kbase device. |
| * @cpu_ts: Output CPU timestamp. |
| * @gpu_ts: Output GPU timestamp. |
| * @gpu_cycle: Output GPU cycle counts. |
| */ |
| static void get_cpu_gpu_time( |
| struct kbase_device *kbdev, |
| u64 *cpu_ts, |
| u64 *gpu_ts, |
| u64 *gpu_cycle) |
| { |
| struct timespec64 ts; |
| |
| kbase_pm_context_active(kbdev); |
| kbase_backend_get_gpu_time(kbdev, gpu_cycle, gpu_ts, &ts); |
| kbase_pm_context_idle(kbdev); |
| |
| if (cpu_ts) |
| *cpu_ts = ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec; |
| } |
| |
| |
| /** |
| * kbase_ts_converter_init() - Initialize system timestamp converter. |
| * |
| * @self: System Timestamp Converter instance. |
| * @kbdev: Kbase device pointer |
| * |
| * Return: Zero on success, -1 otherwise. |
| */ |
| static int kbase_ts_converter_init( |
| struct kbase_ts_converter *self, |
| struct kbase_device *kbdev) |
| { |
| u64 cpu_ts = 0; |
| u64 gpu_ts = 0; |
| u64 freq; |
| u64 common_factor; |
| |
| get_cpu_gpu_time(kbdev, &cpu_ts, &gpu_ts, NULL); |
| freq = arch_timer_get_cntfrq(); |
| |
| if (!freq) { |
| dev_warn(kbdev->dev, "arch_timer_get_rate() is zero!"); |
| return -1; |
| } |
| |
| common_factor = gcd(NSEC_PER_SEC, freq); |
| |
| self->multiplier = div64_u64(NSEC_PER_SEC, common_factor); |
| self->divisor = div64_u64(freq, common_factor); |
| self->offset = |
| cpu_ts - div64_u64(gpu_ts * self->multiplier, self->divisor); |
| |
| return 0; |
| } |
| |
| /** |
| * kbase_ts_converter_convert() - Convert GPU timestamp to CPU timestamp. |
| * |
| * @self: System Timestamp Converter instance. |
| * @gpu_ts: System timestamp value to converter. |
| * |
| * Return: The CPU timestamp. |
| */ |
| static void kbase_ts_converter_convert( |
| const struct kbase_ts_converter *self, |
| u64 *gpu_ts) |
| { |
| u64 old_gpu_ts = *gpu_ts; |
| *gpu_ts = div64_u64(old_gpu_ts * self->multiplier, |
| self->divisor) + self->offset; |
| } |
| |
| /** |
| * tl_reader_overflow_notify() - Emit stream overflow tracepoint. |
| * |
| * @self: CSFFW TL Reader instance. |
| * @msg_buf_start: Start of the message. |
| * @msg_buf_end: End of the message buffer. |
| */ |
| static void tl_reader_overflow_notify( |
| const struct kbase_csf_tl_reader *self, |
| u8 *const msg_buf_start, |
| u8 *const msg_buf_end) |
| { |
| struct kbase_device *kbdev = self->kbdev; |
| struct kbase_csffw_tl_message message = {0}; |
| |
| /* Reuse the timestamp and cycle count from current event if possible */ |
| if (msg_buf_start + sizeof(message) <= msg_buf_end) |
| memcpy(&message, msg_buf_start, sizeof(message)); |
| |
| KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW( |
| kbdev, message.timestamp, message.cycle_counter); |
| } |
| |
| /** |
| * tl_reader_overflow_check() - Check if an overflow has happened |
| * |
| * @self: CSFFW TL Reader instance. |
| * @event_id: Incoming event id. |
| * |
| * Return: True, if an overflow has happened, False otherwise. |
| */ |
| static bool tl_reader_overflow_check( |
| struct kbase_csf_tl_reader *self, |
| u16 event_id) |
| { |
| struct kbase_device *kbdev = self->kbdev; |
| bool has_overflow = false; |
| |
| /* 0 is a special event_id and reserved for the very first tracepoint |
| * after reset, we should skip overflow check when reset happened. |
| */ |
| if (event_id != 0) { |
| has_overflow = self->got_first_event |
| && self->expected_event_id != event_id; |
| |
| if (has_overflow) |
| dev_warn(kbdev->dev, |
| "CSFFW overflow, event_id: %u, expected: %u.", |
| event_id, self->expected_event_id); |
| } |
| |
| self->got_first_event = true; |
| self->expected_event_id = event_id + 1; |
| /* When event_id reaches its max value, it skips 0 and wraps to 1. */ |
| if (self->expected_event_id == 0) |
| self->expected_event_id++; |
| |
| return has_overflow; |
| } |
| |
| /** |
| * tl_reader_reset() - Reset timeline tracebuffer reader state machine. |
| * |
| * @self: CSFFW TL Reader instance. |
| * |
| * Reset the reader to the default state, i.e. set all the |
| * mutable fields to zero. |
| */ |
| static void tl_reader_reset(struct kbase_csf_tl_reader *self) |
| { |
| self->got_first_event = false; |
| self->is_active = false; |
| self->expected_event_id = 0; |
| self->tl_header.btc = 0; |
| } |
| |
| int kbase_csf_tl_reader_flush_buffer(struct kbase_csf_tl_reader *self) |
| { |
| int ret = 0; |
| struct kbase_device *kbdev = self->kbdev; |
| struct kbase_tlstream *stream = self->stream; |
| |
| u8 *read_buffer = self->read_buffer; |
| const size_t read_buffer_size = sizeof(self->read_buffer); |
| |
| u32 bytes_read; |
| u8 *csffw_data_begin; |
| u8 *csffw_data_end; |
| u8 *csffw_data_it; |
| |
| unsigned long flags; |
| |
| spin_lock_irqsave(&self->read_lock, flags); |
| |
| /* If not running, early exit. */ |
| if (!self->is_active) { |
| spin_unlock_irqrestore(&self->read_lock, flags); |
| return -EBUSY; |
| } |
| |
| /* Copying the whole buffer in a single shot. We assume |
| * that the buffer will not contain partially written messages. |
| */ |
| bytes_read = kbase_csf_firmware_trace_buffer_read_data( |
| self->trace_buffer, read_buffer, read_buffer_size); |
| csffw_data_begin = read_buffer; |
| csffw_data_end = read_buffer + bytes_read; |
| |
| for (csffw_data_it = csffw_data_begin; |
| csffw_data_it < csffw_data_end;) { |
| u32 event_header; |
| u16 event_id; |
| u16 event_size; |
| unsigned long acq_flags; |
| char *buffer; |
| |
| /* Can we safely read event_id? */ |
| if (csffw_data_it + sizeof(event_header) > csffw_data_end) { |
| dev_warn( |
| kbdev->dev, |
| "Unable to parse CSFFW tracebuffer event header."); |
| ret = -EBUSY; |
| break; |
| } |
| |
| /* Read and parse the event header. */ |
| memcpy(&event_header, csffw_data_it, sizeof(event_header)); |
| event_id = (event_header >> 0) & 0xFFFF; |
| event_size = (event_header >> 16) & 0xFFFF; |
| csffw_data_it += sizeof(event_header); |
| |
| /* Detect if an overflow has happened. */ |
| if (tl_reader_overflow_check(self, event_id)) |
| tl_reader_overflow_notify(self, |
| csffw_data_it, |
| csffw_data_end); |
| |
| /* Can we safely read the message body? */ |
| if (csffw_data_it + event_size > csffw_data_end) { |
| dev_warn(kbdev->dev, |
| "event_id: %u, can't read with event_size: %u.", |
| event_id, event_size); |
| ret = -EBUSY; |
| break; |
| } |
| |
| /* Convert GPU timestamp to CPU timestamp. */ |
| { |
| struct kbase_csffw_tl_message *msg = |
| (struct kbase_csffw_tl_message *) csffw_data_it; |
| kbase_ts_converter_convert( |
| &self->ts_converter, |
| &msg->timestamp); |
| } |
| |
| /* Copy the message out to the tl_stream. */ |
| buffer = kbase_tlstream_msgbuf_acquire( |
| stream, event_size, &acq_flags); |
| kbasep_serialize_bytes(buffer, 0, csffw_data_it, event_size); |
| kbase_tlstream_msgbuf_release(stream, acq_flags); |
| csffw_data_it += event_size; |
| } |
| |
| spin_unlock_irqrestore(&self->read_lock, flags); |
| return ret; |
| } |
| |
| static void kbasep_csf_tl_reader_read_callback(struct timer_list *timer) |
| { |
| struct kbase_csf_tl_reader *self = |
| container_of(timer, struct kbase_csf_tl_reader, read_timer); |
| |
| int rcode; |
| |
| kbase_csf_tl_reader_flush_buffer(self); |
| |
| rcode = mod_timer(&self->read_timer, |
| jiffies + msecs_to_jiffies(self->timer_interval)); |
| |
| CSTD_UNUSED(rcode); |
| } |
| |
| /** |
| * tl_reader_init_late() - Late CSFFW TL Reader initialization. |
| * |
| * @self: CSFFW TL Reader instance. |
| * @kbdev: Kbase device. |
| * |
| * Late initialization is done once at kbase_csf_tl_reader_start() time. |
| * This is because the firmware image is not parsed |
| * by the kbase_csf_tl_reader_init() time. |
| * |
| * Return: Zero on success, -1 otherwise. |
| */ |
| static int tl_reader_init_late( |
| struct kbase_csf_tl_reader *self, |
| struct kbase_device *kbdev) |
| { |
| struct firmware_trace_buffer *tb; |
| size_t hdr_size = 0; |
| const char *hdr = NULL; |
| |
| if (self->kbdev) |
| return 0; |
| |
| tb = kbase_csf_firmware_get_trace_buffer( |
| kbdev, KBASE_CSFFW_TRACEBUFFER_NAME); |
| hdr = kbase_csf_firmware_get_timeline_metadata( |
| kbdev, KBASE_CSFFW_TIMELINE_HEADER_NAME, &hdr_size); |
| |
| if (!tb) { |
| dev_warn( |
| kbdev->dev, |
| "'%s' tracebuffer is not present in the firmware image.", |
| KBASE_CSFFW_TRACEBUFFER_NAME); |
| return -1; |
| } |
| |
| if (!hdr) { |
| dev_warn( |
| kbdev->dev, |
| "'%s' timeline metadata is not present in the firmware image.", |
| KBASE_CSFFW_TIMELINE_HEADER_NAME); |
| return -1; |
| } |
| |
| if (kbase_ts_converter_init(&self->ts_converter, kbdev)) { |
| return -1; |
| } |
| |
| self->kbdev = kbdev; |
| self->trace_buffer = tb; |
| self->tl_header.data = hdr; |
| self->tl_header.size = hdr_size; |
| |
| return 0; |
| } |
| |
| /** |
| * tl_reader_update_enable_bit() - Update the first bit of a CSFFW tracebuffer. |
| * |
| * @self: CSFFW TL Reader instance. |
| * @value: The value to set. |
| * |
| * Update the first bit of a CSFFW tracebufer and then reset the GPU. |
| * This is to make these changes visible to the MCU. |
| * |
| * Return: 0 on success, or negative error code for failure. |
| */ |
| static int tl_reader_update_enable_bit( |
| struct kbase_csf_tl_reader *self, |
| bool value) |
| { |
| int err = 0; |
| |
| err = kbase_csf_firmware_trace_buffer_update_trace_enable_bit( |
| self->trace_buffer, 0, value); |
| |
| return err; |
| } |
| |
| void kbase_csf_tl_reader_init(struct kbase_csf_tl_reader *self, |
| struct kbase_tlstream *stream) |
| { |
| self->timer_interval = KBASE_CSF_TL_READ_INTERVAL_DEFAULT; |
| |
| kbase_timer_setup(&self->read_timer, |
| kbasep_csf_tl_reader_read_callback); |
| |
| self->stream = stream; |
| |
| /* This will be initialized by tl_reader_init_late() */ |
| self->kbdev = NULL; |
| self->trace_buffer = NULL; |
| self->tl_header.data = NULL; |
| self->tl_header.size = 0; |
| |
| spin_lock_init(&self->read_lock); |
| |
| tl_reader_reset(self); |
| } |
| |
| void kbase_csf_tl_reader_term(struct kbase_csf_tl_reader *self) |
| { |
| del_timer_sync(&self->read_timer); |
| } |
| |
| int kbase_csf_tl_reader_start(struct kbase_csf_tl_reader *self, |
| struct kbase_device *kbdev) |
| { |
| int rcode; |
| |
| /* If already running, early exit. */ |
| if (self->is_active) |
| return 0; |
| |
| if (tl_reader_init_late(self, kbdev)) { |
| return -EINVAL; |
| } |
| |
| tl_reader_reset(self); |
| |
| self->is_active = true; |
| /* Set bytes to copy to the header size. This is to trigger copying |
| * of the header to the user space. |
| */ |
| self->tl_header.btc = self->tl_header.size; |
| |
| /* Enable the tracebuffer on the CSFFW side. */ |
| rcode = tl_reader_update_enable_bit(self, true); |
| if (rcode != 0) |
| return rcode; |
| |
| rcode = mod_timer(&self->read_timer, |
| jiffies + msecs_to_jiffies(self->timer_interval)); |
| |
| return 0; |
| } |
| |
| void kbase_csf_tl_reader_stop(struct kbase_csf_tl_reader *self) |
| { |
| unsigned long flags; |
| |
| /* If is not running, early exit. */ |
| if (!self->is_active) |
| return; |
| |
| /* Disable the tracebuffer on the CSFFW side. */ |
| tl_reader_update_enable_bit(self, false); |
| |
| del_timer_sync(&self->read_timer); |
| |
| spin_lock_irqsave(&self->read_lock, flags); |
| |
| tl_reader_reset(self); |
| |
| spin_unlock_irqrestore(&self->read_lock, flags); |
| } |
| |
| void kbase_csf_tl_reader_reset(struct kbase_csf_tl_reader *self) |
| { |
| u64 gpu_cycle = 0; |
| struct kbase_device *kbdev = self->kbdev; |
| |
| if (!kbdev) |
| return; |
| |
| kbase_csf_tl_reader_flush_buffer(self); |
| |
| get_cpu_gpu_time(kbdev, NULL, NULL, &gpu_cycle); |
| KBASE_TLSTREAM_TL_KBASE_CSFFW_RESET(kbdev, gpu_cycle); |
| } |