| // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "third_party/blink/renderer/modules/webrtc/webrtc_audio_renderer.h" |
| |
| #include <utility> |
| |
| #include "base/location.h" |
| #include "base/logging.h" |
| #include "base/metrics/histogram_macros.h" |
| #include "base/strings/stringprintf.h" |
| #include "base/threading/thread_checker.h" |
| #include "build/build_config.h" |
| #include "media/audio/audio_sink_parameters.h" |
| #include "media/base/audio_bus.h" |
| #include "media/base/audio_capturer_source.h" |
| #include "media/base/audio_latency.h" |
| #include "media/base/audio_parameters.h" |
| #include "media/base/bind_to_current_loop.h" |
| #include "media/base/channel_layout.h" |
| #include "media/base/sample_rates.h" |
| #include "third_party/blink/public/platform/modules/webrtc/webrtc_logging.h" |
| #include "third_party/blink/public/platform/platform.h" |
| #include "third_party/blink/public/web/web_local_frame.h" |
| #include "third_party/blink/public/web/web_local_frame_client.h" |
| #include "third_party/blink/renderer/core/frame/local_frame.h" |
| #include "third_party/blink/renderer/platform/mediastream/media_stream_audio_track.h" |
| #include "third_party/blink/renderer/platform/mediastream/media_stream_component.h" |
| #include "third_party/blink/renderer/platform/scheduler/public/post_cross_thread_task.h" |
| #include "third_party/blink/renderer/platform/scheduler/public/thread.h" |
| #include "third_party/blink/renderer/platform/webrtc/peer_connection_remote_audio_source.h" |
| #include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h" |
| #include "third_party/blink/renderer/platform/wtf/text/string_builder.h" |
| #include "third_party/webrtc/api/media_stream_interface.h" |
| |
| namespace WTF { |
| |
| template <typename T> |
| struct CrossThreadCopier<rtc::scoped_refptr<T>> { |
| STATIC_ONLY(CrossThreadCopier); |
| using Type = rtc::scoped_refptr<T>; |
| static Type Copy(Type pointer) { return pointer; } |
| }; |
| |
| } // namespace WTF |
| |
| namespace blink { |
| |
| namespace { |
| |
| // Audio parameters that don't change. |
| const media::AudioParameters::Format kFormat = |
| media::AudioParameters::AUDIO_PCM_LOW_LATENCY; |
| |
| // Time constant for AudioPowerMonitor. See See AudioPowerMonitor ctor comments |
| // for details. |
| constexpr base::TimeDelta kPowerMeasurementTimeConstant = |
| base::TimeDelta::FromMilliseconds(10); |
| |
| // Time in seconds between two successive measurements of audio power levels. |
| constexpr base::TimeDelta kPowerMonitorLogInterval = |
| base::TimeDelta::FromSeconds(15); |
| |
| // Used for UMA histograms. |
| const int kRenderTimeHistogramMinMicroseconds = 100; |
| const int kRenderTimeHistogramMaxMicroseconds = 1 * 1000 * 1000; // 1 second |
| |
| const char* OutputDeviceStatusToString(media::OutputDeviceStatus status) { |
| switch (status) { |
| case media::OUTPUT_DEVICE_STATUS_OK: |
| return "OK"; |
| case media::OUTPUT_DEVICE_STATUS_ERROR_NOT_FOUND: |
| return "ERROR_NOT_FOUND"; |
| case media::OUTPUT_DEVICE_STATUS_ERROR_NOT_AUTHORIZED: |
| return "ERROR_NOT_AUTHORIZED"; |
| case media::OUTPUT_DEVICE_STATUS_ERROR_TIMED_OUT: |
| return "ERROR_TIMED_OUT"; |
| case media::OUTPUT_DEVICE_STATUS_ERROR_INTERNAL: |
| return "ERROR_INTERNAL"; |
| } |
| } |
| |
| const char* StateToString(WebRtcAudioRenderer::State state) { |
| switch (state) { |
| case WebRtcAudioRenderer::UNINITIALIZED: |
| return "UNINITIALIZED"; |
| case WebRtcAudioRenderer::PLAYING: |
| return "PLAYING"; |
| case WebRtcAudioRenderer::PAUSED: |
| return "PAUSED"; |
| } |
| } |
| |
| // This is a simple wrapper class that's handed out to users of a shared |
| // WebRtcAudioRenderer instance. This class maintains the per-user 'playing' |
| // and 'started' states to avoid problems related to incorrect usage which |
| // might violate the implementation assumptions inside WebRtcAudioRenderer |
| // (see the play reference count). |
| class SharedAudioRenderer : public WebMediaStreamAudioRenderer { |
| public: |
| // Callback definition for a callback that is called when when Play(), Pause() |
| // or SetVolume are called (whenever the internal |playing_state_| changes). |
| using OnPlayStateChanged = |
| base::RepeatingCallback<void(MediaStreamDescriptor*, |
| WebRtcAudioRenderer::PlayingState*)>; |
| |
| // Signals that the PlayingState* is about to become invalid, see comment in |
| // OnPlayStateRemoved. |
| using OnPlayStateRemoved = |
| base::OnceCallback<void(WebRtcAudioRenderer::PlayingState*)>; |
| |
| SharedAudioRenderer( |
| const scoped_refptr<WebMediaStreamAudioRenderer>& delegate, |
| MediaStreamDescriptor* media_stream_descriptor, |
| const OnPlayStateChanged& on_play_state_changed, |
| OnPlayStateRemoved on_play_state_removed) |
| : delegate_(delegate), |
| media_stream_descriptor_(media_stream_descriptor), |
| started_(false), |
| on_play_state_changed_(on_play_state_changed), |
| on_play_state_removed_(std::move(on_play_state_removed)) { |
| DCHECK(!on_play_state_changed_.is_null()); |
| DCHECK(media_stream_descriptor_); |
| } |
| |
| protected: |
| ~SharedAudioRenderer() override { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| DVLOG(1) << __func__; |
| Stop(); |
| std::move(on_play_state_removed_).Run(&playing_state_); |
| } |
| |
| void Start() override { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| if (started_) |
| return; |
| started_ = true; |
| delegate_->Start(); |
| } |
| |
| void Play() override { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| if (!started_ || playing_state_.playing()) |
| return; |
| playing_state_.set_playing(true); |
| on_play_state_changed_.Run(media_stream_descriptor_, &playing_state_); |
| } |
| |
| void Pause() override { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| if (!started_ || !playing_state_.playing()) |
| return; |
| playing_state_.set_playing(false); |
| on_play_state_changed_.Run(media_stream_descriptor_, &playing_state_); |
| } |
| |
| void Stop() override { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| if (!started_) |
| return; |
| Pause(); |
| started_ = false; |
| delegate_->Stop(); |
| } |
| |
| void SetVolume(float volume) override { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| DCHECK(volume >= 0.0f && volume <= 1.0f); |
| playing_state_.set_volume(volume); |
| on_play_state_changed_.Run(media_stream_descriptor_, &playing_state_); |
| } |
| |
| void SwitchOutputDevice(const std::string& device_id, |
| media::OutputDeviceStatusCB callback) override { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| return delegate_->SwitchOutputDevice(device_id, std::move(callback)); |
| } |
| |
| base::TimeDelta GetCurrentRenderTime() override { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| return delegate_->GetCurrentRenderTime(); |
| } |
| |
| bool IsLocalRenderer() override { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| return delegate_->IsLocalRenderer(); |
| } |
| |
| private: |
| THREAD_CHECKER(thread_checker_); |
| const scoped_refptr<WebMediaStreamAudioRenderer> delegate_; |
| Persistent<MediaStreamDescriptor> media_stream_descriptor_; |
| bool started_; |
| WebRtcAudioRenderer::PlayingState playing_state_; |
| OnPlayStateChanged on_play_state_changed_; |
| OnPlayStateRemoved on_play_state_removed_; |
| }; |
| |
| } // namespace |
| |
| class WebRtcAudioRenderer::InternalFrame { |
| public: |
| explicit InternalFrame(WebLocalFrame* web_frame) |
| : frame_(web_frame ? static_cast<LocalFrame*>( |
| WebLocalFrame::ToCoreFrame(*web_frame)) |
| : nullptr) {} |
| |
| LocalFrame* frame() { return frame_.Get(); } |
| WebLocalFrame* web_frame() { |
| if (!frame_) |
| return nullptr; |
| |
| return static_cast<WebLocalFrame*>(WebFrame::FromCoreFrame(frame())); |
| } |
| |
| private: |
| WeakPersistent<LocalFrame> frame_; |
| }; |
| |
| WebRtcAudioRenderer::AudioStreamTracker::AudioStreamTracker( |
| scoped_refptr<base::SingleThreadTaskRunner> task_runner, |
| WebRtcAudioRenderer* renderer, |
| int sample_rate) |
| : task_runner_(std::move(task_runner)), |
| renderer_(renderer), |
| start_time_(base::TimeTicks::Now()), |
| render_callbacks_started_(false), |
| check_alive_timer_(task_runner_, |
| this, |
| &WebRtcAudioRenderer::AudioStreamTracker::CheckAlive), |
| power_monitor_(sample_rate, kPowerMeasurementTimeConstant), |
| last_audio_level_log_time_(base::TimeTicks::Now()) { |
| weak_this_ = weak_factory_.GetWeakPtr(); |
| // CheckAlive() will look to see if |render_callbacks_started_| is true |
| // after the timeout expires and log this. If the stream is paused/closed |
| // before the timer fires, a warning is logged instead. |
| check_alive_timer_.StartOneShot(base::TimeDelta::FromSeconds(5), FROM_HERE); |
| } |
| |
| WebRtcAudioRenderer::AudioStreamTracker::~AudioStreamTracker() { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| DCHECK(renderer_); |
| const auto duration = base::TimeTicks::Now() - start_time_; |
| renderer_->SendLogMessage( |
| String::Format("%s => (media stream duration=%" PRId64 " seconds)", |
| __func__, duration.InSeconds())); |
| } |
| |
| void WebRtcAudioRenderer::AudioStreamTracker::OnRenderCallbackCalled() { |
| DCHECK(renderer_->CurrentThreadIsRenderingThread()); |
| // Indicate that render callbacks has started as expected and within a |
| // reasonable time. Since this thread is the only writer of |
| // |render_callbacks_started_| once the thread starts, it's safe to compare |
| // and then change the state once. |
| if (!render_callbacks_started_) |
| render_callbacks_started_ = true; |
| } |
| |
| void WebRtcAudioRenderer::AudioStreamTracker::MeasurePower( |
| const media::AudioBus& buffer, |
| int frames) { |
| DCHECK(renderer_->CurrentThreadIsRenderingThread()); |
| // Update the average power estimate on the rendering thread to avoid posting |
| // a task which also has to copy the audio bus. According to comments in |
| // AudioPowerMonitor::Scan(), it should be safe. Note that, we only check the |
| // power once every ten seconds (on the |task_runner_| thread) and the result |
| // is only used for logging purposes. |
| power_monitor_.Scan(buffer, frames); |
| const auto now = base::TimeTicks::Now(); |
| if ((now - last_audio_level_log_time_) > kPowerMonitorLogInterval) { |
| // Log the current audio level but avoid using the render thread to reduce |
| // its load and to ensure that |power_monitor_| is mainly accessed on one |
| // thread. |weak_ptr_factory_| ensures that the task is canceled when |
| // |this| is destroyed since we can't guarantee that |this| outlives the |
| // task. |
| PostCrossThreadTask( |
| *task_runner_, FROM_HERE, |
| CrossThreadBindOnce(&AudioStreamTracker::LogAudioPowerLevel, |
| weak_this_)); |
| last_audio_level_log_time_ = now; |
| } |
| } |
| |
| void WebRtcAudioRenderer::AudioStreamTracker::LogAudioPowerLevel() { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| std::pair<float, bool> power_and_clip = |
| power_monitor_.ReadCurrentPowerAndClip(); |
| renderer_->SendLogMessage(String::Format( |
| "%s => (average audio level=%.2f dBFS)", __func__, power_and_clip.first)); |
| } |
| |
| void WebRtcAudioRenderer::AudioStreamTracker::CheckAlive(TimerBase*) { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| DCHECK(renderer_); |
| renderer_->SendLogMessage(String::Format( |
| "%s => (%s)", __func__, |
| render_callbacks_started_ ? "stream is alive" |
| : "WARNING: stream is not alive")); |
| } |
| |
| WebRtcAudioRenderer::WebRtcAudioRenderer( |
| const scoped_refptr<base::SingleThreadTaskRunner>& signaling_thread, |
| MediaStreamDescriptor* media_stream_descriptor, |
| WebLocalFrame* web_frame, |
| const base::UnguessableToken& session_id, |
| const String& device_id, |
| base::RepeatingCallback<void()> on_render_error_callback) |
| : task_runner_(Thread::Current()->GetTaskRunner()), |
| state_(UNINITIALIZED), |
| source_internal_frame_(std::make_unique<InternalFrame>(web_frame)), |
| session_id_(session_id), |
| signaling_thread_(signaling_thread), |
| media_stream_descriptor_(media_stream_descriptor), |
| media_stream_descriptor_id_(media_stream_descriptor_->Id()), |
| source_(nullptr), |
| play_ref_count_(0), |
| start_ref_count_(0), |
| sink_params_(kFormat, media::CHANNEL_LAYOUT_STEREO, 0, 0), |
| output_device_id_(device_id), |
| on_render_error_callback_(std::move(on_render_error_callback)) { |
| if (web_frame && web_frame->Client()) { |
| speech_recognition_client_ = |
| web_frame->Client()->CreateSpeechRecognitionClient( |
| media::BindToCurrentLoop( |
| ConvertToBaseOnceCallback(CrossThreadBindOnce( |
| &WebRtcAudioRenderer::EnableSpeechRecognition, |
| weak_factory_.GetWeakPtr())))); |
| } |
| |
| SendLogMessage( |
| String::Format("%s({session_id=%s}, {device_id=%s})", __func__, |
| session_id.is_empty() ? "" : session_id.ToString().c_str(), |
| device_id.Utf8().c_str())); |
| } |
| |
| WebRtcAudioRenderer::~WebRtcAudioRenderer() { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| DCHECK_EQ(state_, UNINITIALIZED); |
| } |
| |
| bool WebRtcAudioRenderer::Initialize(WebRtcAudioRendererSource* source) { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| DCHECK(source); |
| DCHECK(!sink_.get()); |
| { |
| base::AutoLock auto_lock(lock_); |
| DCHECK_EQ(state_, UNINITIALIZED); |
| DCHECK(!source_); |
| } |
| SendLogMessage( |
| String::Format("%s([state=%s])", __func__, StateToString(state_))); |
| |
| media::AudioSinkParameters sink_params(session_id_, output_device_id_.Utf8()); |
| sink_params.processing_id = source->GetAudioProcessingId(); |
| sink_ = Platform::Current()->NewAudioRendererSink( |
| WebAudioDeviceSourceType::kWebRtc, source_internal_frame_->web_frame(), |
| sink_params); |
| |
| media::OutputDeviceStatus sink_status = |
| sink_->GetOutputDeviceInfo().device_status(); |
| UMA_HISTOGRAM_ENUMERATION("Media.Audio.WebRTCAudioRenderer.DeviceStatus", |
| sink_status, media::OUTPUT_DEVICE_STATUS_MAX + 1); |
| SendLogMessage(String::Format("%s => (sink device_status=%s)", __func__, |
| OutputDeviceStatusToString(sink_status))); |
| if (sink_status != media::OUTPUT_DEVICE_STATUS_OK) { |
| SendLogMessage(String::Format("%s => (ERROR: invalid output device status)", |
| __func__)); |
| sink_->Stop(); |
| return false; |
| } |
| |
| PrepareSink(); |
| { |
| // No need to reassert the preconditions because the other thread |
| // accessing the fields only reads them. |
| base::AutoLock auto_lock(lock_); |
| source_ = source; |
| |
| // User must call Play() before any audio can be heard. |
| state_ = PAUSED; |
| } |
| source_->SetOutputDeviceForAec(output_device_id_); |
| sink_->Start(); |
| sink_->Play(); // Not all the sinks play on start. |
| |
| return true; |
| } |
| |
| scoped_refptr<WebMediaStreamAudioRenderer> |
| WebRtcAudioRenderer::CreateSharedAudioRendererProxy( |
| MediaStreamDescriptor* media_stream_descriptor) { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| SharedAudioRenderer::OnPlayStateChanged on_play_state_changed = |
| WTF::BindRepeating(&WebRtcAudioRenderer::OnPlayStateChanged, |
| WrapRefCounted(this)); |
| SharedAudioRenderer::OnPlayStateRemoved on_play_state_removed = |
| WTF::Bind(&WebRtcAudioRenderer::OnPlayStateRemoved, WrapRefCounted(this)); |
| return new SharedAudioRenderer(this, media_stream_descriptor, |
| std::move(on_play_state_changed), |
| std::move(on_play_state_removed)); |
| } |
| |
| bool WebRtcAudioRenderer::IsStarted() const { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| return start_ref_count_ != 0; |
| } |
| |
| bool WebRtcAudioRenderer::CurrentThreadIsRenderingThread() { |
| return sink_->CurrentThreadIsRenderingThread(); |
| } |
| |
| void WebRtcAudioRenderer::Start() { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| SendLogMessage( |
| String::Format("%s([state=%s])", __func__, StateToString(state_))); |
| ++start_ref_count_; |
| } |
| |
| void WebRtcAudioRenderer::Play() { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| SendLogMessage( |
| String::Format("%s([state=%s])", __func__, StateToString(state_))); |
| if (playing_state_.playing()) |
| return; |
| |
| playing_state_.set_playing(true); |
| |
| OnPlayStateChanged(media_stream_descriptor_, &playing_state_); |
| } |
| |
| void WebRtcAudioRenderer::EnterPlayState() { |
| DVLOG(1) << "WebRtcAudioRenderer::EnterPlayState()"; |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| DCHECK_GT(start_ref_count_, 0) << "Did you forget to call Start()?"; |
| SendLogMessage( |
| String::Format("%s([state=%s])", __func__, StateToString(state_))); |
| base::AutoLock auto_lock(lock_); |
| if (state_ == UNINITIALIZED) |
| return; |
| |
| DCHECK(play_ref_count_ == 0 || state_ == PLAYING); |
| ++play_ref_count_; |
| |
| if (state_ != PLAYING) { |
| state_ = PLAYING; |
| |
| audio_stream_tracker_.emplace(task_runner_, this, |
| sink_params_.sample_rate()); |
| |
| if (audio_fifo_) { |
| audio_delay_ = base::TimeDelta(); |
| audio_fifo_->Clear(); |
| } |
| } |
| SendLogMessage( |
| String::Format("%s => (state=%s)", __func__, StateToString(state_))); |
| } |
| |
| void WebRtcAudioRenderer::Pause() { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| SendLogMessage( |
| String::Format("%s([state=%s])", __func__, StateToString(state_))); |
| if (!playing_state_.playing()) |
| return; |
| |
| playing_state_.set_playing(false); |
| |
| OnPlayStateChanged(media_stream_descriptor_, &playing_state_); |
| } |
| |
| void WebRtcAudioRenderer::EnterPauseState() { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| DCHECK_GT(start_ref_count_, 0) << "Did you forget to call Start()?"; |
| SendLogMessage( |
| String::Format("%s([state=%s])", __func__, StateToString(state_))); |
| base::AutoLock auto_lock(lock_); |
| if (state_ == UNINITIALIZED) |
| return; |
| |
| DCHECK_EQ(state_, PLAYING); |
| DCHECK_GT(play_ref_count_, 0); |
| if (!--play_ref_count_) |
| state_ = PAUSED; |
| SendLogMessage( |
| String::Format("%s => (state=%s)", __func__, StateToString(state_))); |
| } |
| |
| void WebRtcAudioRenderer::Stop() { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| { |
| SendLogMessage( |
| String::Format("%s([state=%s])", __func__, StateToString(state_))); |
| base::AutoLock auto_lock(lock_); |
| if (state_ == UNINITIALIZED) |
| return; |
| |
| if (--start_ref_count_) |
| return; |
| |
| audio_stream_tracker_.reset(); |
| source_->RemoveAudioRenderer(this); |
| source_ = nullptr; |
| state_ = UNINITIALIZED; |
| } |
| |
| // Apart from here, |max_render_time_| is only accessed in SourceCallback(), |
| // which is guaranteed to not run after |source_| has been set to null, and |
| // not before this function has returned. |
| // If |max_render_time_| is zero, no render call has been made. |
| if (!max_render_time_.is_zero()) { |
| UMA_HISTOGRAM_CUSTOM_COUNTS( |
| "Media.Audio.Render.GetSourceDataTimeMax.WebRTC", |
| static_cast<int>(max_render_time_.InMicroseconds()), |
| kRenderTimeHistogramMinMicroseconds, |
| kRenderTimeHistogramMaxMicroseconds, 50); |
| SendLogMessage(String::Format("%s => (max_render_time=%.3f ms)", __func__, |
| max_render_time_.InMillisecondsF())); |
| max_render_time_ = base::TimeDelta(); |
| } |
| |
| // Make sure to stop the sink while _not_ holding the lock since the Render() |
| // callback may currently be executing and trying to grab the lock while we're |
| // stopping the thread on which it runs. |
| sink_->Stop(); |
| } |
| |
| void WebRtcAudioRenderer::SetVolume(float volume) { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| DCHECK(volume >= 0.0f && volume <= 1.0f); |
| SendLogMessage(String::Format("%s({volume=%.2f})", __func__, volume)); |
| |
| playing_state_.set_volume(volume); |
| OnPlayStateChanged(media_stream_descriptor_, &playing_state_); |
| } |
| |
| base::TimeDelta WebRtcAudioRenderer::GetCurrentRenderTime() { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| base::AutoLock auto_lock(lock_); |
| return current_time_; |
| } |
| |
| bool WebRtcAudioRenderer::IsLocalRenderer() { |
| return false; |
| } |
| |
| void WebRtcAudioRenderer::SwitchOutputDevice( |
| const std::string& device_id, |
| media::OutputDeviceStatusCB callback) { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| SendLogMessage(String::Format("%s({device_id=%s} [state=%s])", __func__, |
| device_id.c_str(), StateToString(state_))); |
| if (!source_) { |
| SendLogMessage(String::Format( |
| "%s => (ERROR: OUTPUT_DEVICE_STATUS_ERROR_INTERNAL)", __func__)); |
| std::move(callback).Run(media::OUTPUT_DEVICE_STATUS_ERROR_INTERNAL); |
| return; |
| } |
| |
| { |
| base::AutoLock auto_lock(lock_); |
| DCHECK_NE(state_, UNINITIALIZED); |
| } |
| |
| auto* web_frame = source_internal_frame_->web_frame(); |
| #if !defined(OS_ANDROID) |
| // Frames are allowed to be null in Android due to an issue in tests. |
| // In practice, this is not an issue, since Android does not support |
| // setSinkId(). https://crbug.com/1119689 |
| if (!web_frame) { |
| SendLogMessage(String::Format("%s => (ERROR: No Frame)", __func__)); |
| std::move(callback).Run(media::OUTPUT_DEVICE_STATUS_ERROR_INTERNAL); |
| return; |
| } |
| #endif |
| |
| media::AudioSinkParameters sink_params(session_id_, device_id); |
| sink_params.processing_id = source_->GetAudioProcessingId(); |
| scoped_refptr<media::AudioRendererSink> new_sink = |
| Platform::Current()->NewAudioRendererSink( |
| WebAudioDeviceSourceType::kWebRtc, web_frame, sink_params); |
| media::OutputDeviceStatus status = |
| new_sink->GetOutputDeviceInfo().device_status(); |
| UMA_HISTOGRAM_ENUMERATION( |
| "Media.Audio.WebRTCAudioRenderer.SwitchDeviceStatus", status, |
| media::OUTPUT_DEVICE_STATUS_MAX + 1); |
| SendLogMessage(String::Format("%s => (sink device_status=%s)", __func__, |
| OutputDeviceStatusToString(status))); |
| |
| if (status != media::OUTPUT_DEVICE_STATUS_OK) { |
| SendLogMessage( |
| String::Format("%s => (ERROR: invalid sink device status)", __func__)); |
| new_sink->Stop(); |
| std::move(callback).Run(status); |
| return; |
| } |
| |
| // Make sure to stop the sink while _not_ holding the lock since the Render() |
| // callback may currently be executing and trying to grab the lock while we're |
| // stopping the thread on which it runs. |
| sink_->Stop(); |
| sink_ = new_sink; |
| output_device_id_ = String::FromUTF8(device_id); |
| { |
| base::AutoLock auto_lock(lock_); |
| source_->AudioRendererThreadStopped(); |
| } |
| source_->SetOutputDeviceForAec(output_device_id_); |
| PrepareSink(); |
| sink_->Start(); |
| sink_->Play(); // Not all the sinks play on start. |
| |
| std::move(callback).Run(media::OUTPUT_DEVICE_STATUS_OK); |
| } |
| |
| void WebRtcAudioRenderer::TranscribeAudio( |
| std::unique_ptr<media::AudioBus> audio_bus, |
| int sample_rate, |
| media::ChannelLayout channel_layout) { |
| speech_recognition_client_->AddAudio(std::move(audio_bus), sample_rate, |
| channel_layout); |
| } |
| |
| int WebRtcAudioRenderer::Render(base::TimeDelta delay, |
| base::TimeTicks delay_timestamp, |
| int prior_frames_skipped, |
| media::AudioBus* audio_bus) { |
| DCHECK(sink_->CurrentThreadIsRenderingThread()); |
| DCHECK_LE(sink_params_.channels(), 8); |
| base::AutoLock auto_lock(lock_); |
| if (!source_) |
| return 0; |
| |
| audio_delay_ = delay; |
| |
| // If there are skipped frames, pull and throw away the same amount. We always |
| // pull 10 ms of data from the source (see PrepareSink()), so the fifo is only |
| // required if the number of frames to drop doesn't correspond to 10 ms. |
| if (prior_frames_skipped > 0) { |
| const int source_frames_per_buffer = sink_params_.sample_rate() / 100; |
| if (!audio_fifo_ && prior_frames_skipped != source_frames_per_buffer) { |
| audio_fifo_ = std::make_unique<media::AudioPullFifo>( |
| sink_params_.channels(), source_frames_per_buffer, |
| ConvertToBaseRepeatingCallback( |
| CrossThreadBindRepeating(&WebRtcAudioRenderer::SourceCallback, |
| CrossThreadUnretained(this)))); |
| } |
| |
| std::unique_ptr<media::AudioBus> drop_bus = |
| media::AudioBus::Create(audio_bus->channels(), prior_frames_skipped); |
| if (audio_fifo_) |
| audio_fifo_->Consume(drop_bus.get(), drop_bus->frames()); |
| else |
| SourceCallback(0, drop_bus.get()); |
| } |
| |
| // Pull the data we will deliver. |
| if (audio_fifo_) |
| audio_fifo_->Consume(audio_bus, audio_bus->frames()); |
| else |
| SourceCallback(0, audio_bus); |
| |
| if (state_ == PLAYING && audio_stream_tracker_) { |
| // Mark the stream as alive the first time this method is called. |
| audio_stream_tracker_->OnRenderCallbackCalled(); |
| audio_stream_tracker_->MeasurePower(*audio_bus, audio_bus->frames()); |
| } |
| |
| if (transcribe_audio_callback_) { |
| auto audio_bus_copy = |
| media::AudioBus::Create(audio_bus->channels(), audio_bus->frames()); |
| audio_bus->CopyTo(audio_bus_copy.get()); |
| transcribe_audio_callback_.Run(std::move(audio_bus_copy), |
| sink_params_.sample_rate(), |
| sink_params_.channel_layout()); |
| } |
| |
| return (state_ == PLAYING) ? audio_bus->frames() : 0; |
| } |
| |
| void WebRtcAudioRenderer::OnRenderError() { |
| DCHECK(on_render_error_callback_); |
| PostCrossThreadTask( |
| *task_runner_, FROM_HERE, |
| CrossThreadBindOnce(&WebRtcAudioRenderer::OnRenderErrorCrossThread, |
| WrapRefCounted(this))); |
| } |
| |
| void WebRtcAudioRenderer::OnRenderErrorCrossThread() { |
| DCHECK(task_runner_->BelongsToCurrentThread()); |
| |
| on_render_error_callback_.Run(); |
| } |
| |
| // Called by AudioPullFifo when more data is necessary. |
| void WebRtcAudioRenderer::SourceCallback(int fifo_frame_delay, |
| media::AudioBus* audio_bus) { |
| DCHECK(sink_->CurrentThreadIsRenderingThread()); |
| base::TimeTicks start_time = base::TimeTicks::Now(); |
| DVLOG(2) << "WRAR::SourceCallback(" << fifo_frame_delay << ", " |
| << audio_bus->channels() << ", " << audio_bus->frames() << ")"; |
| |
| int output_delay_milliseconds = |
| static_cast<int>(audio_delay_.InMilliseconds()); |
| // TODO(grunell): This integer division by sample_rate will cause loss of |
| // partial milliseconds, and may cause avsync drift. http://crbug.com/586540 |
| output_delay_milliseconds += fifo_frame_delay * |
| base::Time::kMillisecondsPerSecond / |
| sink_params_.sample_rate(); |
| DVLOG(2) << "output_delay_milliseconds: " << output_delay_milliseconds; |
| |
| // We need to keep render data for the |source_| regardless of |state_|, |
| // otherwise the data will be buffered up inside |source_|. |
| source_->RenderData(audio_bus, sink_params_.sample_rate(), |
| output_delay_milliseconds, ¤t_time_); |
| |
| // Avoid filling up the audio bus if we are not playing; instead |
| // return here and ensure that the returned value in Render() is 0. |
| if (state_ != PLAYING) |
| audio_bus->Zero(); |
| |
| // Measure the elapsed time for this function and log it to UMA. Store the max |
| // value. Don't do this for low resolution clocks to not skew data. |
| if (base::TimeTicks::IsHighResolution()) { |
| base::TimeDelta elapsed = base::TimeTicks::Now() - start_time; |
| UMA_HISTOGRAM_CUSTOM_COUNTS("Media.Audio.Render.GetSourceDataTime.WebRTC", |
| static_cast<int>(elapsed.InMicroseconds()), |
| kRenderTimeHistogramMinMicroseconds, |
| kRenderTimeHistogramMaxMicroseconds, 50); |
| |
| if (elapsed > max_render_time_) |
| max_render_time_ = elapsed; |
| } |
| } |
| |
| void WebRtcAudioRenderer::UpdateSourceVolume( |
| webrtc::AudioSourceInterface* source) { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| |
| // Note: If there are no playing audio renderers, then the volume will be |
| // set to 0.0. |
| float volume = 0.0f; |
| |
| auto entry = source_playing_states_.find(source); |
| if (entry != source_playing_states_.end()) { |
| PlayingStates& states = entry->second; |
| for (PlayingStates::const_iterator it = states.begin(); it != states.end(); |
| ++it) { |
| if ((*it)->playing()) |
| volume += (*it)->volume(); |
| } |
| } |
| |
| // The valid range for volume scaling of a remote webrtc source is |
| // 0.0-10.0 where 1.0 is no attenuation/boost. |
| DCHECK(volume >= 0.0f); |
| if (volume > 10.0f) |
| volume = 10.0f; |
| |
| SendLogMessage(String::Format("%s => (source volume changed to %.2f)", |
| __func__, volume)); |
| if (!signaling_thread_->BelongsToCurrentThread()) { |
| // Libjingle hands out proxy objects in most cases, but the audio source |
| // object is an exception (bug?). So, to work around that, we need to make |
| // sure we call SetVolume on the signaling thread. |
| PostCrossThreadTask( |
| *signaling_thread_, FROM_HERE, |
| CrossThreadBindOnce( |
| &webrtc::AudioSourceInterface::SetVolume, |
| rtc::scoped_refptr<webrtc::AudioSourceInterface>(source), volume)); |
| } else { |
| source->SetVolume(volume); |
| } |
| } |
| |
| bool WebRtcAudioRenderer::AddPlayingState(webrtc::AudioSourceInterface* source, |
| PlayingState* state) { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| DCHECK(state->playing()); |
| // Look up or add the |source| to the map. |
| PlayingStates& array = source_playing_states_[source]; |
| if (base::Contains(array, state)) |
| return false; |
| |
| array.push_back(state); |
| SendLogMessage(String::Format("%s => (number of playing audio sources=%d)", |
| __func__, static_cast<int>(array.size()))); |
| |
| return true; |
| } |
| |
| bool WebRtcAudioRenderer::RemovePlayingState( |
| webrtc::AudioSourceInterface* source, |
| PlayingState* state) { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| DCHECK(!state->playing()); |
| auto found = source_playing_states_.find(source); |
| if (found == source_playing_states_.end()) |
| return false; |
| |
| PlayingStates& array = found->second; |
| auto state_it = std::find(array.begin(), array.end(), state); |
| if (state_it == array.end()) |
| return false; |
| |
| array.erase(state_it); |
| |
| if (array.empty()) |
| source_playing_states_.erase(found); |
| |
| return true; |
| } |
| |
| void WebRtcAudioRenderer::OnPlayStateChanged( |
| MediaStreamDescriptor* media_stream_descriptor, |
| PlayingState* state) { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| const HeapVector<Member<MediaStreamComponent>>& components = |
| media_stream_descriptor->AudioComponents(); |
| |
| for (auto component : components) { |
| // WebRtcAudioRenderer can only render audio tracks received from a remote |
| // peer. Since the actual MediaStream is mutable from JavaScript, we need |
| // to make sure |component| is actually a remote track. |
| PeerConnectionRemoteAudioTrack* const remote_track = |
| PeerConnectionRemoteAudioTrack::From( |
| MediaStreamAudioTrack::From(component.Get())); |
| if (!remote_track) |
| continue; |
| webrtc::AudioSourceInterface* source = |
| remote_track->track_interface()->GetSource(); |
| DCHECK(source); |
| if (!state->playing()) { |
| if (RemovePlayingState(source, state)) |
| EnterPauseState(); |
| } else if (AddPlayingState(source, state)) { |
| EnterPlayState(); |
| } |
| UpdateSourceVolume(source); |
| } |
| } |
| |
| void WebRtcAudioRenderer::OnPlayStateRemoved(PlayingState* state) { |
| // It is possible we associated |state| to a source for a track that is no |
| // longer easily reachable. We iterate over |source_playing_states_| to |
| // ensure there are no dangling pointers to |state| there. See |
| // crbug.com/697256. |
| // TODO(maxmorin): Clean up cleanup code in this and related classes so that |
| // this hack isn't necessary. |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| for (auto it = source_playing_states_.begin(); |
| it != source_playing_states_.end();) { |
| PlayingStates& states = it->second; |
| // We cannot use RemovePlayingState as it might invalidate |it|. |
| base::Erase(states, state); |
| if (states.empty()) |
| it = source_playing_states_.erase(it); |
| else |
| ++it; |
| } |
| } |
| |
| void WebRtcAudioRenderer::PrepareSink() { |
| DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); |
| SendLogMessage(String::Format("%s()", __func__)); |
| media::AudioParameters new_sink_params; |
| { |
| base::AutoLock lock(lock_); |
| new_sink_params = sink_params_; |
| } |
| |
| const media::OutputDeviceInfo& device_info = sink_->GetOutputDeviceInfo(); |
| DCHECK_EQ(device_info.device_status(), media::OUTPUT_DEVICE_STATUS_OK); |
| SendLogMessage(String::Format( |
| "%s => (hardware parameters=[%s])", __func__, |
| device_info.output_params().AsHumanReadableString().c_str())); |
| |
| // WebRTC does not yet support higher rates than 192000 on the client side |
| // and 48000 is the preferred sample rate. Therefore, if 192000 is detected, |
| // we change the rate to 48000 instead. The consequence is that the native |
| // layer will be opened up at 192kHz but WebRTC will provide data at 48kHz |
| // which will then be resampled by the audio converted on the browser side |
| // to match the native audio layer. |
| int sample_rate = device_info.output_params().sample_rate(); |
| if (sample_rate >= 192000) { |
| SendLogMessage( |
| String::Format("%s => (WARNING: WebRTC provides audio at 48kHz and " |
| "resampling takes place to match %dHz)", |
| __func__, sample_rate)); |
| sample_rate = 48000; |
| } |
| media::AudioSampleRate asr; |
| if (media::ToAudioSampleRate(sample_rate, &asr)) { |
| UMA_HISTOGRAM_ENUMERATION("WebRTC.AudioOutputSampleRate", asr, |
| media::kAudioSampleRateMax + 1); |
| } else { |
| UMA_HISTOGRAM_COUNTS_1M("WebRTC.AudioOutputSampleRateUnexpected", |
| sample_rate); |
| } |
| |
| // Calculate the frames per buffer for the source, i.e. the WebRTC client. We |
| // use 10 ms of data since the WebRTC client only supports multiples of 10 ms |
| // as buffer size where 10 ms is preferred for lowest possible delay. |
| const int source_frames_per_buffer = (sample_rate / 100); |
| SendLogMessage(String::Format("%s => (source_frames_per_buffer=%d)", __func__, |
| source_frames_per_buffer)); |
| |
| // Setup sink parameters using same channel configuration as the source. |
| // This sink is an AudioRendererSink which is implemented by an |
| // AudioOutputDevice. Note that we used to use hard-coded settings for |
| // stereo here but this has been changed since crbug.com/982276. |
| constexpr int kMaxChannels = 8; |
| int channels = device_info.output_params().channels(); |
| media::ChannelLayout channel_layout = |
| device_info.output_params().channel_layout(); |
| if (channels > kMaxChannels) { |
| // WebRTC does not support channel remixing for more than 8 channels (7.1). |
| // This is an attempt to "support" more than 8 channels by falling back to |
| // stereo instead. See crbug.com/1003735. |
| SendLogMessage( |
| String::Format("%s => (WARNING: sink falls back to stereo)", __func__)); |
| channels = 2; |
| channel_layout = media::CHANNEL_LAYOUT_STEREO; |
| } |
| const int sink_frames_per_buffer = media::AudioLatency::GetRtcBufferSize( |
| sample_rate, device_info.output_params().frames_per_buffer()); |
| new_sink_params.Reset(kFormat, channel_layout, sample_rate, |
| sink_frames_per_buffer); |
| if (channel_layout == media::CHANNEL_LAYOUT_DISCRETE) { |
| new_sink_params.set_channels_for_discrete(channels); |
| } |
| DCHECK(new_sink_params.IsValid()); |
| |
| // Create a FIFO if re-buffering is required to match the source input with |
| // the sink request. The source acts as provider here and the sink as |
| // consumer. |
| const bool different_source_sink_frames = |
| source_frames_per_buffer != new_sink_params.frames_per_buffer(); |
| if (different_source_sink_frames) { |
| SendLogMessage(String::Format("%s => (INFO: rebuffering from %d to %d)", |
| __func__, source_frames_per_buffer, |
| new_sink_params.frames_per_buffer())); |
| } |
| { |
| base::AutoLock lock(lock_); |
| if ((!audio_fifo_ && different_source_sink_frames) || |
| (audio_fifo_ && |
| audio_fifo_->SizeInFrames() != source_frames_per_buffer)) { |
| audio_fifo_ = std::make_unique<media::AudioPullFifo>( |
| channels, source_frames_per_buffer, |
| ConvertToBaseRepeatingCallback( |
| CrossThreadBindRepeating(&WebRtcAudioRenderer::SourceCallback, |
| CrossThreadUnretained(this)))); |
| } |
| sink_params_ = new_sink_params; |
| SendLogMessage( |
| String::Format("%s => (sink_params=[%s])", __func__, |
| sink_params_.AsHumanReadableString().c_str())); |
| } |
| |
| // Specify the latency info to be passed to the browser side. |
| new_sink_params.set_latency_tag( |
| Platform::Current()->GetAudioSourceLatencyType( |
| WebAudioDeviceSourceType::kWebRtc)); |
| sink_->Initialize(new_sink_params, this); |
| } |
| |
| void WebRtcAudioRenderer::SendLogMessage(const WTF::String& message) { |
| WebRtcLogMessage(String::Format("WRAR::%s [label=%s]", message.Utf8().c_str(), |
| media_stream_descriptor_id_.Utf8().c_str()) |
| .Utf8()); |
| } |
| |
| void WebRtcAudioRenderer::EnableSpeechRecognition() { |
| if (speech_recognition_client_ && |
| speech_recognition_client_->IsSpeechRecognitionAvailable()) { |
| transcribe_audio_callback_ = |
| media::BindToCurrentLoop(ConvertToBaseRepeatingCallback( |
| CrossThreadBindRepeating(&WebRtcAudioRenderer::TranscribeAudio, |
| weak_factory_.GetWeakPtr()))); |
| } |
| } |
| |
| } // namespace blink |