I’m getting other player’s voice chat from Vivox using participant taps and pushing them into Unity Audio Sources, then getting the data with OnAudioFilterRead and pushing it to FMOD.
I managed to get this working by following this thread, but I’m out of my depth here.
The implementation is mostly there, but the voice has some stuttering and or crackling to it, which seems to be somewhat inconsistent. I’ve managed to get it pretty low by experimenting with different buffer sizes in FMOD and Unity, since the audio also goes through Unity’s audio system but the issue is still noticeable. The audio is crystal clear when coming out of the audio source, but not when it goes in and comes out of FMOD.
Also, I have the programmer instrument set as Async inside FMOD. Are there any setup that should be done in FMOD that I might be missing?
Here’s the script.
public class AudioFilterReader : MonoBehaviour
{
private int systemSamplerate;
private FMOD.Studio.EVENT_CALLBACK audioCallback;
private Vector3 worldPos;
VivoxParticipant participant;
AudioOccluder myOccluder;
private Queue<EventInstance> eventInstancePool = new Queue<EventInstance>();
private void Start()
{
systemSamplerate = AudioSettings.outputSampleRate;
InitializeEventInstancePool(100);
}
private void InitializeEventInstancePool(int size)
{
for (int i = 0; i < size; i++)
{
EventInstance instance = RuntimeManager.CreateInstance("event:/char/otherPlayers/VOIP");
instance.set3DAttributes(worldPos.To3DAttributes());
eventInstancePool.Enqueue(instance);
}
}
private EventInstance GetPooledEventInstance()
{
if (eventInstancePool.Count > 0)
{
return eventInstancePool.Dequeue();
}
else
{
UnityEngine.Debug.LogWarning("EventInstance pool is empty. Creating a new instance.");
return RuntimeManager.CreateInstance("event:/char/otherPlayers/VOIP");
}
}
private void ReturnEventInstanceToPool(EventInstance instance)
{
eventInstancePool.Enqueue(instance);
}
private void OnAudioFilterRead(float[] data, int channels)
{
if (data == null || data.Length == 0)
return;
if (!participant.SpeechDetected) return;
EventInstance programmerSound = GetPooledEventInstance();
programmerSound.set3DAttributes(worldPos.To3DAttributes());
var result = ConvertArrayToFMODSound(data, channels, out var sound);
if (result != RESULT.OK)
{
UnityEngine.Debug.LogError($"Error creating the sound {result}");
return;
}
GCHandle soundHandle = GCHandle.Alloc(sound, GCHandleType.Pinned);
programmerSound.setUserData(GCHandle.ToIntPtr(soundHandle));
audioCallback = new FMOD.Studio.EVENT_CALLBACK(AudioEventCallback);
programmerSound.setCallback(audioCallback);
programmerSound.start();
programmerSound.setParameterByName("Occlusion", myOccluder.GetCurrentOcclusion());
ReturnEventInstanceToPool(programmerSound);
}
private RESULT ConvertArrayToFMODSound(float[] data, int channels, out Sound sound)
{
uint lenBytes = (uint)(data.Length * sizeof(float));
CREATESOUNDEXINFO soundInfo = new CREATESOUNDEXINFO();
soundInfo.length = lenBytes;
soundInfo.format = SOUND_FORMAT.PCMFLOAT;
soundInfo.numchannels = channels;
soundInfo.defaultfrequency = systemSamplerate;
soundInfo.cbsize = Marshal.SizeOf(typeof(FMOD.CREATESOUNDEXINFO));
var res = RuntimeManager.CoreSystem.createSound("voip", MODE.OPENUSER, ref soundInfo, out sound);
if (systemSamplerate == 0)
{
UnityEngine.Debug.LogError("System sample rate is 0");
sound.release();
return RESULT.ERR_FORMAT;
}
if (res != RESULT.OK)
{
UnityEngine.Debug.LogError($"Result is not valid. {res}");
sound.release();
return res;
}
IntPtr ptr1, ptr2;
uint len1, len2;
sound.@lock(0, lenBytes, out ptr1, out ptr2, out len1, out len2);
Marshal.Copy(data, 0, ptr1, (int)(len1 / sizeof(float)));
if (len2 > 0)
{
Marshal.Copy(data, (int)(len1 / sizeof(float)), ptr2, (int)(len2 / sizeof(float)));
}
sound.unlock(ptr1, ptr2, len1, len2);
var result = sound.setMode(MODE.OPENUSER);
return result;
}
private void Update()
{
worldPos = transform.position;
if (participant.SpeechDetected)
myOccluder.calculate = true;
else
myOccluder.calculate = false;
}
[MonoPInvokeCallback(typeof(EVENT_CALLBACK))]
static RESULT AudioEventCallback(EVENT_CALLBACK_TYPE type, IntPtr instancePtr, IntPtr parameterPtr)
{
EventInstance instance = new EventInstance(instancePtr);
// Retrieve the user data
instance.getUserData(out var pointer);
// Get the string object
GCHandle stringHandle = GCHandle.FromIntPtr(pointer);
Sound sound = (Sound)stringHandle.Target;
switch (type)
{
case EVENT_CALLBACK_TYPE.CREATE_PROGRAMMER_SOUND:
{
var parameter = (PROGRAMMER_SOUND_PROPERTIES)Marshal.PtrToStructure(parameterPtr, typeof(PROGRAMMER_SOUND_PROPERTIES));
parameter.sound = sound.handle;
parameter.subsoundIndex = -1;
Marshal.StructureToPtr(parameter, parameterPtr, false);
break;
}
case EVENT_CALLBACK_TYPE.DESTROY_PROGRAMMER_SOUND:
{
var parameter = (PROGRAMMER_SOUND_PROPERTIES)Marshal.PtrToStructure(parameterPtr, typeof(PROGRAMMER_SOUND_PROPERTIES));
sound.release();
sound = new Sound(parameter.sound);
sound.release();
break;
}
case EVENT_CALLBACK_TYPE.DESTROYED:
{
// Now the event has been destroyed, unpin the string memory so it can be garbage collected
stringHandle.Free();
break;
}
}
return RESULT.OK;
}
private void OnDestroy()
{
foreach (var eventInstance in eventInstancePool)
{
eventInstance.release();
}
eventInstancePool.Clear();
}
}
