Hi Connor
Thanks for your reply.
Your reply made me realised that although the mic recording is setup via code, it’s not a programmer sound! I have combined the two scripts which I will paste below.
This works fairly well although I am having a ~300ms delay even when I reduce the latency values as far as I can [Edit: latency was solved by adjusting DSP buffer values in FMOD settings].
Also I enter a “domain reloading” loop once every few play-mode exits which requires me to end-task unity.
If you have any insight into either of these then please let me know, otherwise the code is here as a start point for anyone else doing the same:
using FMOD;
public class MicRecording
{
private readonly MicSettings micSettings;
private readonly int driverId;
public readonly int NativeRate;
public readonly int NativeChannels;
private uint samplesRecorded;
private uint samplesPlayed;
private uint recSoundLength;
private uint lastPlayPos;
private uint lastRecordPos;
private uint adjustLatency;
private int actualLatency;
private uint minRecordDelta;
private Sound recSound;
private ChannelGroup recChannelGroup;
private uint DriftThreshold => (uint)(NativeRate * micSettings.driftThresholdMS) / 1000;
private uint DesiredLatency => (uint)(NativeRate * micSettings.targetLatencyMS) / 1000;
public MicRecording(MicSettings micSettings, int driverId)
{
this.driverId = driverId;
this.micSettings = micSettings;
/*
Determine latency in samples.
*/
FMODUnity.RuntimeManager.CoreSystem.getRecordDriverInfo(driverId, out _, 0, out _, out NativeRate, out _, out NativeChannels, out _);
adjustLatency = DesiredLatency;
actualLatency = (int)DesiredLatency;
}
public void Initialise(Sound sound, ChannelGroup channelGroup)
{
recSound = sound;
recChannelGroup = channelGroup;
/*
Create user sound to record into, then start recording.
*/
FMODUnity.RuntimeManager.CoreSystem.recordStart(driverId, recSound, true);
recSound.getLength(out recSoundLength, TIMEUNIT.PCM);
}
public void Tick()
{
/*
Determine how much has been recorded since we last checked
*/
FMODUnity.RuntimeManager.CoreSystem.getRecordPosition(driverId, out uint recordPos);
uint recordDelta = (recordPos >= lastRecordPos)
? (recordPos - lastRecordPos)
: (recordPos + recSoundLength - lastRecordPos);
lastRecordPos = recordPos;
samplesRecorded += recordDelta;
//uint minRecordDelta = 0;
if (recordDelta != 0 && (recordDelta < minRecordDelta))
{
minRecordDelta = recordDelta; // Smallest driver granularity seen so far
adjustLatency =
(recordDelta <= DesiredLatency)
? DesiredLatency
: recordDelta; // Adjust our latency if driver granularity is high
}
/*
Determine how much has been played since we last checked.
*/
recChannelGroup.getNumChannels(out int channelCount);
if (channelCount == 0)
{
return;
}
if (recChannelGroup.getChannel(0, out Channel recChannel) != RESULT.OK)
{
return;
}
if (recChannel.hasHandle())
{
recChannel.getPosition(out uint playPos, TIMEUNIT.PCM);
uint playDelta = (playPos >= lastPlayPos)
? (playPos - lastPlayPos)
: (playPos + recSoundLength - lastPlayPos);
lastPlayPos = playPos;
samplesPlayed += playDelta;
// Compensate for any drift.
int latency = (int)(samplesRecorded - samplesPlayed);
actualLatency = (int)((0.97f * actualLatency) + (0.03f * latency));
int playbackRate = NativeRate;
bool slow = actualLatency < (int)(adjustLatency - DriftThreshold);
bool faster = actualLatency > (int)(adjustLatency + DriftThreshold);
if (slow)
{
// Playback position is catching up to the record position, slow playback down by 2%
playbackRate = NativeRate - (NativeRate / 50);
}
else if (faster)
{
// Playback is falling behind the record position, speed playback up by 2%
playbackRate = NativeRate + (NativeRate / 50);
}
recChannel.setFrequency(playbackRate);
}
}
public void StopRecording()
{
FMODUnity.RuntimeManager.CoreSystem.recordStop(0);
}
}
using System;
using UnityEngine;
using System.Runtime.InteropServices;
using FMOD;
using FMOD.Studio;
using FMODUnity;
using Debug = UnityEngine.Debug;
class ScriptUsageProgrammerSounds : MonoBehaviour
{
[SerializeField] private MicSettings micSettings;
[SerializeField] private int driverId;
public EventReference eventReference;
private EVENT_CALLBACK micRecordingCallback;
private MicRecording micRecording;
private static Action<string, ChannelGroup, IntPtr> _startMic;
private static Action _stopMic;
private void Start()
{
// Explicitly create the delegate object and assign it to a member so it doesn't get freed
// by the garbage collected while it's being used
micRecordingCallback = new (MicRecordingEventCallback);
EventInstance soundInstance = RuntimeManager.CreateInstance(eventReference.Path);
// Pin the key string in memory and pass a pointer through the user data
GCHandle stringHandle = GCHandle.Alloc("MicSound");
soundInstance.setUserData(GCHandle.ToIntPtr(stringHandle));
soundInstance.setCallback(micRecordingCallback);
soundInstance.start();
soundInstance.release();
// Previous MicRecordingEventCallback was not static and invoked these but this caused unity to crash
// Using static actions resolve this, see https://blog.kylekukshtel.com/fmod-crashing-programmer-sound-callback-unity-native
_startMic = StartMic;
_stopMic = StopMic;
}
private void LateUpdate()
{
micRecording?.Tick();
}
[AOT.MonoPInvokeCallback(typeof(EVENT_CALLBACK))]
private static RESULT MicRecordingEventCallback(EVENT_CALLBACK_TYPE type, IntPtr instancePtr, IntPtr parameterPtr)
{
Debug.Log($"[MicRecordingEventCallback] callback type = {type}, instancePtr = {instancePtr}");
EventInstance instance = new(instancePtr);
// Retrieve the user data
instance.getUserData(out IntPtr stringPtr);
// Get the string object
GCHandle stringHandle = GCHandle.FromIntPtr(stringPtr);
string key = stringHandle.Target as string;
switch (type)
{
case EVENT_CALLBACK_TYPE.CREATE_PROGRAMMER_SOUND:
{
instance.getChannelGroup(out ChannelGroup channelGroup);
_startMic(key, channelGroup, parameterPtr);
break;
}
case EVENT_CALLBACK_TYPE.DESTROY_PROGRAMMER_SOUND:
{
PROGRAMMER_SOUND_PROPERTIES parameter = (PROGRAMMER_SOUND_PROPERTIES)Marshal.PtrToStructure(parameterPtr, typeof(PROGRAMMER_SOUND_PROPERTIES));
Sound sound = new(parameter.sound);
sound.release();
_stopMic();
break;
}
case EVENT_CALLBACK_TYPE.DESTROYED:
{
// Now the event has been destroyed, unpin the string memory so it can be garbage collected
stringHandle.Free();
break;
}
}
return RESULT.OK;
}
private void StartMic(string key, ChannelGroup channelGroup, IntPtr parameterPtr)
{
micRecording = new MicRecording(micSettings, driverId);
CREATESOUNDEXINFO createSoundExInfo = new()
{
cbsize = Marshal.SizeOf(typeof(CREATESOUNDEXINFO)),
numchannels = micRecording.NativeChannels,
format = SOUND_FORMAT.PCM16,
defaultfrequency = micRecording.NativeRate,
length = (uint)(micRecording.NativeRate * sizeof(short) * micRecording.NativeChannels)
};
Sound sound = CreateSound(key, parameterPtr, createSoundExInfo);
micRecording.Initialise(sound, channelGroup);
}
private static Sound CreateSound(string key, IntPtr parameterPtr, CREATESOUNDEXINFO createSoundExInfo)
{
MODE soundMode = MODE.LOOP_NORMAL | MODE.OPENUSER;
PROGRAMMER_SOUND_PROPERTIES parameter = (PROGRAMMER_SOUND_PROPERTIES)Marshal.PtrToStructure(parameterPtr, typeof(PROGRAMMER_SOUND_PROPERTIES));
RESULT soundResult = RuntimeManager.CoreSystem.createSound(key, soundMode, ref createSoundExInfo, out Sound sound);
if (soundResult == RESULT.OK)
{
parameter.sound = sound.handle;
parameter.subsoundIndex = -1;
Marshal.StructureToPtr(parameter, parameterPtr, false);
}
else
{
Debug.LogError($"Cannot create mic sound: {soundResult}");
}
return sound;
}
private void StopMic()
{
micRecording.StopRecording();
micRecording = null;
}
}
using System;
[Serializable]
public class MicSettings
{
public uint targetLatencyMS = 5;
public uint driftThresholdMS = 1;
}