Hello, I am very new to FMOD Engine so apologies and thanks in advance.
I am trying to create a Unity game that is basically a mini DAW or a very simplified DAW.
Players should be able to visualize a track’s waveform as well as manipulate the audio in certain ways.
Everywhere online I see visualizations that update every frame, using a FFT DSP to display spectrum data as the audio plays.
What I need is to create a 2D waveform of the sound/track/event, and be able to use that event for further manipulations/processing.
What is working:
- I can successfully analyze an audio track and create a 2D waveform texture. (With some hacks using Sound object, more on that below)
- I can set up an event instance to further manipulate the audio.
What is NOT working:
-
I can only create this 2D waveform by creating a separate Sound object using createSound(filepath) and the filepath of that audio track. Meaning it is not being created from the event. This is cumbersome and not ideal as it involves keeping track of a regular audio file path and a FMOD Studio event.
-
I can instead use an FMOD event, but can’t create a “static” 2D waveform, as the only way I have been able to visualize the event is by displaying the spectrum data on each frame update.
This is ideally what I want to visualize, but like I said can only do this with a created Sound object.
Here are the relevant parts of my main script:
private void Start()
{
// Initialize the sprite renderer.
sprend = this.GetComponent<SpriteRenderer>();
//Prepare FMOD event, sets _event.
PrepareFMODEventInstance();
_samples = new float[_windowSize];
// Get the waveform and add it to the sprite renderer.
Texture2D texwav = GetWaveformFMOD();
Rect rect = new Rect(Vector2.zero, new Vector2(width, height));
sprend.sprite = Sprite.Create(texwav, rect, Vector2.zero);
// Rest of start ...
}
private Texture2D GetWaveformFMOD()
{
int halfheight = height / 2;
float heightscale = (float)halfheight * 0.0025f;
// get the sound data
Texture2D tex = new Texture2D(width, height, TextureFormat.RGBA32, false);
waveform = new float[width];
// THIS IS WHERE IM LIMITED BY USING AN AUDIO FILE PATH.
samples = GetSampleData("Assets/music 1.mp3");
samplesize = samples.Length;
// Debug log to check if the is valid and has data
UnityEngine.Debug.Log("Samples: " + samples);
int packsize = (samplesize / width);
for (int w = 0; w < width; w++)
{
waveform[w] = Mathf.Abs(samples[w * packsize]);
}
// Debug log to check the dimensions and content of the waveform array
UnityEngine.Debug.Log("Waveform array length: " + waveform.Length);
// map the sound data to texture
// 1 - clear
for (int x = 0; x < width; x++)
{
for (int y = 0; y < height; y++)
{
tex.SetPixel(x, y, background);
}
}
// 2 - plot
for (int x = 0; x < width; x++)
{
for (int y = 0; y < waveform[x] * heightscale; y++)
{
tex.SetPixel(x, halfheight + y, foreground);
tex.SetPixel(x, halfheight - y, foreground);
}
}
tex.Apply();
// Debug log to check if the texture is being created and modified as expected
UnityEngine.Debug.Log("Waveform texture created: " + tex.width + "x" + tex.height);
return tex;
}
private byte[] GetSampleData(string filePath)
{
// Very useful tool for debugging FMOD function calls
FMOD.RESULT result;
// Sound variable to retrieve the sample data from
FMOD.Sound sound;
// Creating the sound using the file path of the audio source
// Make sure to create the sound using the MODE.CREATESAMEPLE | MDOE.OPENONLY so the sample data can be retrieved
result = FMODUnity.RuntimeManager.CoreSystem.createSound(filePath, FMOD.MODE.CREATESAMPLE | FMOD.MODE.OPENONLY, out sound);
// Debug the results of the FMOD function call to make sure it got called properly
if (result != FMOD.RESULT.OK)
{
UnityEngine.Debug.Log("Failed to create sound with the result of: " + result);
return null;
}
// Retrieving the length of the sound in milliseconds to size the arrays correctly
result = sound.getLength(out uint length, FMOD.TIMEUNIT.MS);
if (result != FMOD.RESULT.OK)
{
UnityEngine.Debug.Log("Failed to retrieve the length of the sound with result: " + result);
return null;
}
// Creating the return array which will have the sample data is a readable variable type
// Using the length of the sound to create it to the right size
byte[] byteArray = new byte[(int)length];
// Retrieving the sample data to the pointer using the full length of the sound
result = sound.readData(byteArray);
if (result != FMOD.RESULT.OK)
{
UnityEngine.Debug.Log("Failed to retrieve data from sound: " + result);
return null;
}
UnityEngine.Debug.Log("Returning byte array of samples, result: " + result);
//Returning the array populated with the sample data to be used
return byteArray;
}
// Prepare FMOD Event function for added context.
private void PrepareFMODEventInstance()
{
// Create the event instance from the event path, add 3D sound and start.
_event = FMODUnity.RuntimeManager.CreateInstance(_eventPath);
_event.set3DAttributes(FMODUnity.RuntimeUtils.To3DAttributes(gameObject.transform));
//_event.start();
// Create the FFT dsp, set window type, and window size.
FMODUnity.RuntimeManager.CoreSystem.createDSPByType(FMOD.DSP_TYPE.FFT, out _dsp);
_dsp.setParameterInt((int)FMOD.DSP_FFT.WINDOWTYPE, (int)_windowShape);
_dsp.setParameterInt((int)FMOD.DSP_FFT.WINDOWSIZE, _windowSize * 2);
// Get the channel group from the event and add to DSP.
_event.getChannelGroup(out _channelGroup);
_channelGroup.addDSP(0, _dsp);
}
I feel like I am close to what I need, but I am a bit confused and not as knowledgeable about DSP and fmod in general. Am I just doing this all wrong and can a 2D “static” waveform be simply created using FFT spectrum data?
Thanks again for any help.