448 lines
20 KiB
C#
448 lines
20 KiB
C#
|
using System;
|
|||
|
using UnityEngine;
|
|||
|
using Normal.Realtime.Native;
|
|||
|
using Normal.Realtime.Serialization;
|
|||
|
using Normal.Utility;
|
|||
|
|
|||
|
namespace Normal.Realtime {
|
|||
|
[ExecutionOrder(-1)] // Make sure our Update() runs before the default to ensure _microphoneDbLevel has been calculated for CalculateVoiceVolume()
|
|||
|
public class RealtimeAvatarVoice : RealtimeComponent {
|
|||
|
public float voiceVolume { get; private set; }
|
|||
|
|
|||
|
private RealtimeAvatarVoiceModel _model;
|
|||
|
public RealtimeAvatarVoiceModel model { get { return _model; } set { SetModel(value); } }
|
|||
|
|
|||
|
// TODO: Remove this once we have the equivalent of view.ownedLocally and view.ownedLocallyInhierarchy
|
|||
|
private bool isOwnedLocally {
|
|||
|
get {
|
|||
|
RealtimeView view = realtimeView;
|
|||
|
while (view.isChildView) {
|
|||
|
if (view.isOwnedLocally)
|
|||
|
return true;
|
|||
|
|
|||
|
view = view.transform.parent.GetComponent<RealtimeView>();
|
|||
|
}
|
|||
|
return view.isOwnedLocally;
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
private bool _mute = false;
|
|||
|
public bool mute { get { return _mute; } set { SetMute(value); } }
|
|||
|
|
|||
|
private bool _hasMicrophone { get { return (_oculusMicrophoneDevice != null || _nativeMicrophoneDevice != null || _unityMicrophoneDevice != null); } }
|
|||
|
private OculusMicrophoneDevice _oculusMicrophoneDevice;
|
|||
|
private Native.Microphone _nativeMicrophoneDevice;
|
|||
|
private MicrophoneDevice _unityMicrophoneDevice;
|
|||
|
private AudioDeviceDataReader _unityMicrophoneDeviceDataReader;
|
|||
|
private int _microphoneSampleRate;
|
|||
|
private int _microphoneChannels;
|
|||
|
private int _microphoneFrameSize;
|
|||
|
private AudioInputStream _microphoneStream;
|
|||
|
|
|||
|
private AudioPreprocessor _audioPreprocessor;
|
|||
|
private AudioPreprocessorPlaybackListener _audioPreprocessorPlaybackListener;
|
|||
|
private float _microphoneDbLevel = -42.0f;
|
|||
|
|
|||
|
private AudioOutput _audioOutput;
|
|||
|
|
|||
|
void Update() {
|
|||
|
// Send microphone data if needed
|
|||
|
SendMicrophoneData();
|
|||
|
|
|||
|
// Calculate voice volume level
|
|||
|
CalculateVoiceVolume();
|
|||
|
}
|
|||
|
|
|||
|
void OnDestroy() {
|
|||
|
if (_model != null)
|
|||
|
_model.clientIDStreamIDUpdated -= ClientIDStreamIDUpdated;
|
|||
|
DisconnectAudioStream();
|
|||
|
}
|
|||
|
|
|||
|
private void CalculateVoiceVolume() {
|
|||
|
float averageDbSample = -42.0f;
|
|||
|
|
|||
|
if (_hasMicrophone) {
|
|||
|
averageDbSample = _microphoneDbLevel;
|
|||
|
} else if (_audioOutput != null) {
|
|||
|
averageDbSample = _audioOutput._dbLevel;
|
|||
|
}
|
|||
|
|
|||
|
// These are arbitrary values I picked from my own testing.
|
|||
|
float volumeMinDb = -42.0f;
|
|||
|
float volumeMaxDb = -5.0f;
|
|||
|
float volumeRange = volumeMaxDb - volumeMinDb;
|
|||
|
|
|||
|
float normalizedVolume = (averageDbSample - volumeMinDb) / volumeRange;
|
|||
|
if (normalizedVolume < 0.0f)
|
|||
|
normalizedVolume = 0.0f;
|
|||
|
if (normalizedVolume > 1.0f)
|
|||
|
normalizedVolume = 1.0f;
|
|||
|
|
|||
|
voiceVolume = normalizedVolume;
|
|||
|
}
|
|||
|
|
|||
|
void SetModel(RealtimeAvatarVoiceModel model) {
|
|||
|
if (_model != null)
|
|||
|
_model.clientIDStreamIDUpdated -= ClientIDStreamIDUpdated;
|
|||
|
DisconnectAudioStream();
|
|||
|
|
|||
|
_model = model;
|
|||
|
|
|||
|
ConnectAudioStream();
|
|||
|
if (_model != null)
|
|||
|
_model.clientIDStreamIDUpdated += ClientIDStreamIDUpdated;
|
|||
|
}
|
|||
|
|
|||
|
void ClientIDStreamIDUpdated(RealtimeAvatarVoiceModel model) {
|
|||
|
if (isOwnedLocally)
|
|||
|
return;
|
|||
|
|
|||
|
// Connect the new audio stream
|
|||
|
ConnectAudioStream();
|
|||
|
}
|
|||
|
|
|||
|
void ConnectAudioStream() {
|
|||
|
// Delete the old audio stream
|
|||
|
DisconnectAudioStream();
|
|||
|
|
|||
|
if (_model == null)
|
|||
|
return;
|
|||
|
|
|||
|
if (isOwnedLocally) {
|
|||
|
// Local player, create microphone stream
|
|||
|
|
|||
|
// First check if this platform supports our native microphone wrapper (lower latency + native echo cancellation if available)
|
|||
|
_microphoneSampleRate = 48000;
|
|||
|
_microphoneChannels = 1;
|
|||
|
|
|||
|
// Check for Oculus native microphone device API
|
|||
|
bool foundOculusMicrophoneDevice = false;
|
|||
|
if (OculusMicrophoneDevice.IsOculusPlatformAvailable()) {
|
|||
|
foundOculusMicrophoneDevice = OculusMicrophoneDevice.IsOculusPlatformInitialized();
|
|||
|
if (!foundOculusMicrophoneDevice && Application.platform == RuntimePlatform.Android)
|
|||
|
Debug.LogWarning("Normcore: Oculus Platform SDK found, but it's not initialized. Oculus Quest native echo cancellation will be unavailable.");
|
|||
|
}
|
|||
|
|
|||
|
if (foundOculusMicrophoneDevice) {
|
|||
|
// Create Oculus microphone device
|
|||
|
_oculusMicrophoneDevice = new OculusMicrophoneDevice();
|
|||
|
_oculusMicrophoneDevice.Start();
|
|||
|
_microphoneSampleRate = 48000;
|
|||
|
_microphoneChannels = 1;
|
|||
|
} else if (Native.Microphone.PlatformSupported()) {
|
|||
|
_nativeMicrophoneDevice = new Native.Microphone();
|
|||
|
|
|||
|
// If we failed to connect to the local microphone, bail.
|
|||
|
if (!_nativeMicrophoneDevice.Start()) {
|
|||
|
Debug.LogError("Failed to connect to default microphone device. Make sure it is plugged in and functioning properly.");
|
|||
|
_nativeMicrophoneDevice.Dispose();
|
|||
|
_nativeMicrophoneDevice = null;
|
|||
|
return;
|
|||
|
}
|
|||
|
|
|||
|
_microphoneSampleRate = _nativeMicrophoneDevice.SampleRate();
|
|||
|
_microphoneChannels = _nativeMicrophoneDevice.Channels();
|
|||
|
} else {
|
|||
|
// Create a microphone device
|
|||
|
_unityMicrophoneDevice = MicrophoneDevice.Start("");
|
|||
|
|
|||
|
// If we failed to connect to the local microphone, bail.
|
|||
|
if (_unityMicrophoneDevice == null) {
|
|||
|
Debug.LogError("Failed to connect to default microphone device. Make sure it is plugged in and functioning properly.");
|
|||
|
return;
|
|||
|
}
|
|||
|
|
|||
|
_unityMicrophoneDeviceDataReader = new AudioDeviceDataReader(_unityMicrophoneDevice);
|
|||
|
_microphoneSampleRate = _unityMicrophoneDevice.sampleRate;
|
|||
|
_microphoneChannels = _unityMicrophoneDevice.numberOfChannels;
|
|||
|
}
|
|||
|
|
|||
|
// Compute frame size with the sample rate of the microphone we received
|
|||
|
_microphoneFrameSize = _microphoneSampleRate / 100;
|
|||
|
|
|||
|
// Create microphone stream with this sample rate (stream will automatically resample to 48000 before encoding with OPUS)
|
|||
|
_microphoneStream = room.CreateAudioInputStream(true, _microphoneSampleRate, _microphoneChannels);
|
|||
|
|
|||
|
// Audio Preprocessor
|
|||
|
bool createAudioPreprocessor = Application.platform != RuntimePlatform.IPhonePlayer; // Create it for all platforms except iOS. iOS provides a nice built-in one.
|
|||
|
|
|||
|
if (createAudioPreprocessor) {
|
|||
|
// Turn on echo cancellation for mobile devices;
|
|||
|
bool echoCancellation = Application.isMobilePlatform && Application.platform != RuntimePlatform.IPhonePlayer;
|
|||
|
_audioPreprocessor = new AudioPreprocessor(_microphoneSampleRate,_microphoneFrameSize, // Input stream
|
|||
|
true, // Automatic gain control
|
|||
|
true, // Noise suppression
|
|||
|
true, // Reverb suppression
|
|||
|
echoCancellation, AudioSettings.outputSampleRate, 2, 0.28f); // Echo cancellation
|
|||
|
if (echoCancellation) {
|
|||
|
// Find the audio listener in the scene so we can perform echo cancellation with it
|
|||
|
AudioListener[] audioListeners = FindObjectsOfType<AudioListener>();
|
|||
|
if (audioListeners.Length <= 0) {
|
|||
|
Debug.LogWarning("RealtimeAvatarVoice: Unable to find any AudioListeners in the scene. RealtimeAvatarVoice will not be able to perform echo cancellation.");
|
|||
|
} else {
|
|||
|
AudioListener audioListener = audioListeners[0];
|
|||
|
if (audioListeners.Length > 1)
|
|||
|
Debug.LogWarning("RealtimeAvatarVoice: Multiple AudioListeners found in the scene. Performing echo cancellation with the first one: " + audioListener.gameObject.name);
|
|||
|
|
|||
|
_audioPreprocessorPlaybackListener = audioListener.gameObject.AddComponent<AudioPreprocessorPlaybackListener>();
|
|||
|
_audioPreprocessorPlaybackListener.audioPreprocessor = _audioPreprocessor;
|
|||
|
}
|
|||
|
}
|
|||
|
}
|
|||
|
} else {
|
|||
|
// Remote player, lookup audio stream and create audio output
|
|||
|
int clientID = _model.clientID;
|
|||
|
int streamID = _model.streamID;
|
|||
|
if (clientID >= 0 && streamID >= 0) {
|
|||
|
// Find AudioOutputStream
|
|||
|
AudioOutputStream audioOutputStream = room.GetAudioOutputStream(clientID, streamID);
|
|||
|
if (audioOutputStream != null) {
|
|||
|
_audioOutput = gameObject.AddComponent<AudioOutput>();
|
|||
|
_audioOutput.mute = mute;
|
|||
|
_audioOutput.StartWithAudioOutputStream(audioOutputStream);
|
|||
|
} else {
|
|||
|
Debug.LogError("RealtimeAvatarVoice: Unable to find matching audio stream for avatar (clientID: " + clientID + ", streamID: " + streamID + ").");
|
|||
|
}
|
|||
|
}
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
void DisconnectAudioStream() {
|
|||
|
if (_microphoneStream != null) {
|
|||
|
// Destroy AudioPreprocessorPlaybackListener
|
|||
|
if (_audioPreprocessorPlaybackListener != null) {
|
|||
|
Destroy(_audioPreprocessorPlaybackListener);
|
|||
|
_audioPreprocessorPlaybackListener = null;
|
|||
|
}
|
|||
|
|
|||
|
// Dispose of audio preprocessor
|
|||
|
if (_audioPreprocessor != null) {
|
|||
|
_audioPreprocessor.Dispose();
|
|||
|
_audioPreprocessor = null;
|
|||
|
}
|
|||
|
|
|||
|
// Close microphone stream
|
|||
|
_microphoneStream.Close();
|
|||
|
|
|||
|
// Dispose microphone device
|
|||
|
if (_oculusMicrophoneDevice != null) {
|
|||
|
_oculusMicrophoneDevice.Stop();
|
|||
|
_oculusMicrophoneDevice.Dispose();
|
|||
|
_oculusMicrophoneDevice = null;
|
|||
|
}
|
|||
|
if (_nativeMicrophoneDevice != null) {
|
|||
|
_nativeMicrophoneDevice.Stop();
|
|||
|
_nativeMicrophoneDevice.Dispose();
|
|||
|
_nativeMicrophoneDevice = null;
|
|||
|
}
|
|||
|
if (_unityMicrophoneDevice != null) {
|
|||
|
_unityMicrophoneDevice.Dispose();
|
|||
|
_unityMicrophoneDevice = null;
|
|||
|
}
|
|||
|
|
|||
|
// Clean up
|
|||
|
_unityMicrophoneDeviceDataReader = null;
|
|||
|
_microphoneStream = null;
|
|||
|
}
|
|||
|
|
|||
|
// Remove audio output
|
|||
|
if (_audioOutput != null) {
|
|||
|
_audioOutput.Stop();
|
|||
|
Destroy(_audioOutput);
|
|||
|
_audioOutput = null;
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
void SendMicrophoneData() {
|
|||
|
if (_microphoneStream == null)
|
|||
|
return;
|
|||
|
|
|||
|
// Store the client ID / stream ID so remote clients can find the corresponding AudioOutputStream
|
|||
|
_model.clientID = _microphoneStream.ClientID();
|
|||
|
_model.streamID = _microphoneStream.StreamID();
|
|||
|
|
|||
|
// Check if AudioInputStream is valid.
|
|||
|
if (_model.clientID < 0 || _model.streamID < 0)
|
|||
|
return;
|
|||
|
|
|||
|
// Send audio data in _microphoneFrameSize chunks until we're out of microphone data to send
|
|||
|
float[] audioData = new float[_microphoneFrameSize];
|
|||
|
bool didGetAudioData = false;
|
|||
|
while (GetMicrophoneAudioData(audioData)) {
|
|||
|
// If we have an _audioPreprocessor, preprocess the microphone data to remove noise and echo
|
|||
|
if (_audioPreprocessor != null)
|
|||
|
_audioPreprocessor.ProcessRecordSamples(audioData);
|
|||
|
|
|||
|
// TODO: This is a lame hack. Ideally I'd like to stop sending audio data all together.
|
|||
|
// Note that even when muted audio still needs to run through the audio processor to make sure echo cancellation works properly when mute is turned back off.
|
|||
|
if (_mute)
|
|||
|
Array.Clear(audioData, 0, audioData.Length);
|
|||
|
|
|||
|
// Send out microphone data
|
|||
|
_microphoneStream.SendRawAudioData(audioData);
|
|||
|
|
|||
|
didGetAudioData = true;
|
|||
|
}
|
|||
|
|
|||
|
// If we got audio data, update the current microphone level.
|
|||
|
// Note: I moved this here so that we do our volume level calculations on microphone audio that has run through the AudioPreprocessor.
|
|||
|
if (didGetAudioData) {
|
|||
|
int firstFrame = audioData.Length - 256;
|
|||
|
if (firstFrame < 0)
|
|||
|
firstFrame = 0;
|
|||
|
int firstSample = firstFrame * _microphoneChannels;
|
|||
|
_microphoneDbLevel = StaticFunctions.CalculateAverageDbForAudioBuffer(audioData, firstSample);
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
private bool GetMicrophoneAudioData(float[] audioData) {
|
|||
|
if (_oculusMicrophoneDevice != null)
|
|||
|
return _oculusMicrophoneDevice.GetAudioData(audioData);
|
|||
|
if (_nativeMicrophoneDevice != null)
|
|||
|
return _nativeMicrophoneDevice.GetAudioData(audioData);
|
|||
|
else if (_unityMicrophoneDeviceDataReader != null)
|
|||
|
return _unityMicrophoneDeviceDataReader.GetData(audioData);
|
|||
|
|
|||
|
return false;
|
|||
|
}
|
|||
|
|
|||
|
void SetMute(bool mute) {
|
|||
|
if (mute == _mute)
|
|||
|
return;
|
|||
|
|
|||
|
if (_audioOutput != null)
|
|||
|
_audioOutput.mute = mute;
|
|||
|
|
|||
|
_mute = mute;
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
public class RealtimeAvatarVoiceModel : IModel {
|
|||
|
public delegate void ClientIDStreamIDUpdated(RealtimeAvatarVoiceModel model);
|
|||
|
public event ClientIDStreamIDUpdated clientIDStreamIDUpdated;
|
|||
|
|
|||
|
// TODO: In the future, get the clientID by traversing up to the parent model until we find a metamodel with an owner ID.
|
|||
|
public int clientID {
|
|||
|
get { return _cache.LookForValueInCache(_clientID, entry => entry.clientIDSet, entry => entry.clientID); }
|
|||
|
set { if (value == clientID) return; _cache.UpdateLocalCache(entry => { entry.clientIDSet = true; entry.clientID = value; return entry; }); }
|
|||
|
}
|
|||
|
public int streamID {
|
|||
|
get { return _cache.LookForValueInCache(_streamID, entry => entry.streamIDSet, entry => entry.streamID); }
|
|||
|
set { if (value == streamID) return; _cache.UpdateLocalCache(entry => { entry.streamIDSet = true; entry.streamID = value; return entry; }); }
|
|||
|
}
|
|||
|
|
|||
|
private int _clientID = -1;
|
|||
|
private int _streamID = -1;
|
|||
|
|
|||
|
// Change Cache
|
|||
|
private struct LocalCacheEntry {
|
|||
|
public bool clientIDSet;
|
|||
|
public int clientID;
|
|||
|
public bool streamIDSet;
|
|||
|
public int streamID;
|
|||
|
}
|
|||
|
private LocalChangeCache<LocalCacheEntry> _cache;
|
|||
|
|
|||
|
public RealtimeAvatarVoiceModel() {
|
|||
|
_cache = new LocalChangeCache<LocalCacheEntry>();
|
|||
|
}
|
|||
|
|
|||
|
// Serialization
|
|||
|
public int WriteLength(StreamContext context) {
|
|||
|
int length = 0;
|
|||
|
|
|||
|
if (context.fullModel) {
|
|||
|
// Flatten cache
|
|||
|
_clientID = clientID;
|
|||
|
_streamID = streamID;
|
|||
|
_cache.Clear();
|
|||
|
|
|||
|
// ClientID/StreamID
|
|||
|
length += WriteStream.WriteVarint32Length(1, WriteStream.ConvertNegativeOneIntToUInt(_clientID));
|
|||
|
length += WriteStream.WriteVarint32Length(2, WriteStream.ConvertNegativeOneIntToUInt(_streamID));
|
|||
|
} else {
|
|||
|
// ClientID/StreamID
|
|||
|
if (context.reliableChannel) {
|
|||
|
LocalCacheEntry entry = _cache.localCache;
|
|||
|
if (entry.clientIDSet)
|
|||
|
length += WriteStream.WriteVarint32Length(1, WriteStream.ConvertNegativeOneIntToUInt(entry.clientID));
|
|||
|
if (entry.streamIDSet)
|
|||
|
length += WriteStream.WriteVarint32Length(2, WriteStream.ConvertNegativeOneIntToUInt(entry.streamID));
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
return length;
|
|||
|
}
|
|||
|
|
|||
|
public void Write(WriteStream stream, StreamContext context) {
|
|||
|
if (context.fullModel) {
|
|||
|
// ClientID/StreamID
|
|||
|
stream.WriteVarint32(1, WriteStream.ConvertNegativeOneIntToUInt(_clientID));
|
|||
|
stream.WriteVarint32(2, WriteStream.ConvertNegativeOneIntToUInt(_streamID));
|
|||
|
} else {
|
|||
|
// ClientID/StreamID
|
|||
|
if (context.reliableChannel) {
|
|||
|
// If we're going to send an update. Push the cache to inflight.
|
|||
|
LocalCacheEntry entry = _cache.localCache;
|
|||
|
if (entry.clientIDSet || entry.streamIDSet)
|
|||
|
_cache.PushLocalCacheToInflight(context.updateID);
|
|||
|
|
|||
|
if (entry.clientIDSet)
|
|||
|
stream.WriteVarint32(1, WriteStream.ConvertNegativeOneIntToUInt(entry.clientID));
|
|||
|
if (entry.streamIDSet)
|
|||
|
stream.WriteVarint32(2, WriteStream.ConvertNegativeOneIntToUInt(entry.streamID));
|
|||
|
}
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
public void Read(ReadStream stream, StreamContext context) {
|
|||
|
bool clientIDStreamIDUpdateExistsInCache = _cache.ValueExistsInCache(entry => entry.clientIDSet || entry.streamIDSet);
|
|||
|
|
|||
|
// Remove from in-flight
|
|||
|
if (context.deltaUpdatesOnly && context.reliableChannel)
|
|||
|
_cache.RemoveUpdateFromInflight(context.updateID);
|
|||
|
|
|||
|
bool receivedClientIDStreamIDUpdate = false;
|
|||
|
|
|||
|
// Deserialize
|
|||
|
uint propertyID;
|
|||
|
while (stream.ReadNextPropertyID(out propertyID)) {
|
|||
|
switch (propertyID) {
|
|||
|
case 1:
|
|||
|
// Deserialize value
|
|||
|
int clientID = ReadStream.ConvertUIntToNegativeOneInt(stream.ReadVarint32());
|
|||
|
|
|||
|
// Fire off a notification if it changed
|
|||
|
if (clientID != _clientID)
|
|||
|
receivedClientIDStreamIDUpdate = true;
|
|||
|
|
|||
|
// Store
|
|||
|
_clientID = clientID;
|
|||
|
break;
|
|||
|
case 2:
|
|||
|
// Deserialize value
|
|||
|
int streamID = ReadStream.ConvertUIntToNegativeOneInt(stream.ReadVarint32());
|
|||
|
|
|||
|
// Fire off a notification if it changed
|
|||
|
if (streamID != _streamID)
|
|||
|
receivedClientIDStreamIDUpdate = true;
|
|||
|
|
|||
|
// Store
|
|||
|
_streamID = streamID;
|
|||
|
break;
|
|||
|
default:
|
|||
|
stream.SkipProperty();
|
|||
|
break;
|
|||
|
}
|
|||
|
}
|
|||
|
|
|||
|
// If we received a new value, and there's currently nothing in the cache that trumps this change, fire an update.
|
|||
|
if (receivedClientIDStreamIDUpdate && !clientIDStreamIDUpdateExistsInCache) {
|
|||
|
if (clientIDStreamIDUpdated != null)
|
|||
|
clientIDStreamIDUpdated(this);
|
|||
|
}
|
|||
|
}
|
|||
|
}
|
|||
|
}
|