Hi everyone,
We are working on a game. Our aim is to access in real-time buffer by buffer the samples incoming from a microphone capture. We leveraged the custom DSP example from the API and attached the DSP to a channel group that receives input samples coming from the microphone in-buffer. Here is the code:
// The CustomDSPCallback code was adapted from: https://fmod.com/resources/documentation-unity?version=2.02&page=examples-dsp-capture.html
using System;
using FMODUnity;
using UnityEngine;
using System.Runtime.InteropServices;
public class mic_in : MonoBehaviour
{
//public variables
[Header("Capture Device details")]
public int captureDeviceIndex = 0;
[TextArea] public string captureDeviceName = null;
FMOD.CREATESOUNDEXINFO exinfo;
// Custom DSPCallback variables
private FMOD.DSP_READCALLBACK mReadCallback;
private FMOD.DSP mCaptureDSP;
public float[] mDataBuffer;
private GCHandle mObjHandle;
private uint mBufferLength;
[AOT.MonoPInvokeCallback(typeof(FMOD.DSP_READCALLBACK))]
static FMOD.RESULT CaptureDSPReadCallback(ref FMOD.DSP_STATE dsp_state, IntPtr inbuffer, IntPtr outbuffer, uint length, int inchannels, ref int outchannels)
{
FMOD.DSP_STATE_FUNCTIONS functions = (FMOD.DSP_STATE_FUNCTIONS)Marshal.PtrToStructure(dsp_state.functions, typeof(FMOD.DSP_STATE_FUNCTIONS));
IntPtr userData;
functions.getuserdata(ref dsp_state, out userData);
GCHandle objHandle = GCHandle.FromIntPtr(userData);
mic_in obj = objHandle.Target as mic_in;
Debug.Log("inchannels:"+inchannels);
Debug.Log("outchannels:"+outchannels);
// Copy the incoming buffer to process later
int lengthElements = (int)length * inchannels;
Marshal.Copy(inbuffer, obj.mDataBuffer, 0, lengthElements);
// Copy the inbuffer to the outbuffer so we can still hear it
Marshal.Copy(obj.mDataBuffer, 0, outbuffer, lengthElements);
return FMOD.RESULT.OK;
}
// Start is called before the first frame update
void Start()
{
// how many capture devices are plugged in for us to use.
int numOfDriversConnected;
int numofDrivers;
FMOD.RESULT res = RuntimeManager.CoreSystem.getRecordNumDrivers(out numofDrivers, out numOfDriversConnected);
if(res != FMOD.RESULT.OK)
{
Debug.Log("Failed to retrieve driver details: " + res);
return;
}
if(numOfDriversConnected == 0)
{
Debug.Log("No capture devices detected!");
return;
}
else
Debug.Log("You have " + numOfDriversConnected + " capture devices available to record with.");
// info about the device we're recording with.
System.Guid micGUID;
FMOD.DRIVER_STATE driverState;
FMOD.SPEAKERMODE speakerMode;
int captureSrate;
int captureNumChannels;
RuntimeManager.CoreSystem.getRecordDriverInfo(captureDeviceIndex, out captureDeviceName, 50,
out micGUID, out captureSrate, out speakerMode, out captureNumChannels, out driverState);
Debug.Log("captureNumChannels of capture device: " + captureNumChannels);
Debug.Log("captureSrate: " + captureSrate);
// create sound where capture is recorded
FMOD.Sound sound;
exinfo.cbsize = System.Runtime.InteropServices.Marshal.SizeOf(typeof(FMOD.CREATESOUNDEXINFO));
exinfo.numchannels = captureNumChannels;
exinfo.format = FMOD.SOUND_FORMAT.PCM16;
exinfo.defaultfrequency = captureSrate;
exinfo.length = (uint)captureSrate * sizeof(short) * (uint)captureNumChannels;
RuntimeManager.CoreSystem.createSound(exinfo.userdata, FMOD.MODE.LOOP_NORMAL | FMOD.MODE.OPENUSER,
ref exinfo, out sound);
// start recording
RuntimeManager.CoreSystem.recordStart(captureDeviceIndex, sound, true);
// play sound on dedicated channel in master channel group
FMOD.ChannelGroup masterCG;
FMOD.Channel channel;
if (FMODUnity.RuntimeManager.CoreSystem.getMasterChannelGroup(out masterCG) != FMOD.RESULT.OK)
Debug.LogWarningFormat("FMOD: Unable to create a master channel group: masterCG");
FMODUnity.RuntimeManager.CoreSystem.getMasterChannelGroup(out masterCG);
RuntimeManager.CoreSystem.playSound(sound, masterCG, true, out channel);
channel.setPaused(false);
// Assign the callback to a member variable to avoid garbage collection
mReadCallback = CaptureDSPReadCallback;
// Allocate a data buffer large enough for 8 channels, pin the memory to avoid garbage collection
uint bufferLength;
int numBuffers;
FMODUnity.RuntimeManager.CoreSystem.getDSPBufferSize(out bufferLength, out numBuffers);
mDataBuffer = new float[bufferLength * 8];
mBufferLength = bufferLength;
// Tentatively changed buffer length by calling setDSPBufferSize in file Assets/Plugins/FMOD/src/RuntimeManager.cs
// Tentatively changed buffer length by calling setDSPBufferSize in file Assets/Plugins/FMOD/src/fmod.cs - line 1150
Debug.Log("buffer length:" + bufferLength);
// Get a handle to this object to pass into the callback
mObjHandle = GCHandle.Alloc(this);
if (mObjHandle != null)
{
// Define a basic DSP that receives a callback each mix to capture audio
FMOD.DSP_DESCRIPTION desc = new FMOD.DSP_DESCRIPTION();
desc.numinputbuffers = 1;
desc.numoutputbuffers = 1;
desc.read = mReadCallback;
desc.userdata = GCHandle.ToIntPtr(mObjHandle);
// Create an instance of the capture DSP and attach it to the master channel group to capture all audio
if (FMODUnity.RuntimeManager.CoreSystem.createDSP(ref desc, out mCaptureDSP) == FMOD.RESULT.OK)
{
if (masterCG.addDSP(0, mCaptureDSP) != FMOD.RESULT.OK)
{
Debug.LogWarningFormat("FMOD: Unable to add mCaptureDSP to the master channel group");
}
}
else
{
Debug.LogWarningFormat("FMOD: Unable to create a DSP: mCaptureDSP");
}
}
else
{
Debug.LogWarningFormat("FMOD: Unable to create a GCHandle: mObjHandle");
}
}
}
It sort of works except 2 main issues:
1. There is an echo. DSP is a passthrough copying the inbuffer to the out buffer, but every sounds is creating an echo (repeated twice) even with headphones
2. The latency is huge, and doesn’t seem to be affected at all by the size of the audio buffer set in FMOD settings. Any idea on how we could fix this problem.
Maybe a stream solution is better for the capture of the sound?
Side note: Even if the mic is mono the custom DSP receives 6 channels. We would understand 2 (stereo output), but why the extra 4 channels, which are always silent?
Any help or insights would be much appreciated!
kwe
