Programmer Instrument with Unity WebGL not loading dialogue

Unity
,
1

Hi FMOD Community,

I’m currently working on a Unity project where I’m using FMOD’s programmer sounds feature. Everything works perfectly in the Unity editor and Windows builds, but I’m encountering issues with the WebGL build. Specifically, the programmer sounds do not play, while all other FMOD sounds work fine. I’m loading all the required banks and all voice files are in subfolders in the StreamingAssets folder of the WebGL build.
From what I understand, I need to load the files using WebRequest, however, I can’t seem to get it to work. Code below. Appreciate the help!

using System;
using System.Collections.Generic;
using UnityEngine;
using System.Runtime.InteropServices;
using FMOD;
using FMOD.Studio;
using UnityEngine.Networking;
using UnityEngine.Serialization;
using Debug = FMOD.Debug;

class VoiceHandler : MonoBehaviour
{
    FMOD.Studio.EVENT_CALLBACK dialogueCallback;

    [SerializeField] private string generalEventName = "event:/Voiceover/Talk_General";
    [SerializeField] private string levelEventName = "event:/Voiceover/Talk_X1";


    [SerializeField] private string[] generalBankPrefixes = { "WRONG", "NUM" };

    private EventInstance _dialogueInstance;

    void Start()
    {
        // Explicitly create the delegate object and assign it to a member so it doesn't get freed
        // by the garbage collected while it's being used
        dialogueCallback = DialogueEventCallback;
    }

    public void PlayDialogue(string key)
    {
        _dialogueInstance =
            FMODUnity.RuntimeManager.CreateInstance((key.StartsWith("GENERAL"))
                ? generalEventName
                : levelEventName);

        // Pin the key string in memory and pass a pointer through the user data
        GCHandle stringHandle = GCHandle.Alloc(key);
        _dialogueInstance.setUserData(GCHandle.ToIntPtr(stringHandle));

        _dialogueInstance.setCallback(dialogueCallback);
        _dialogueInstance.start();
        _dialogueInstance.release();
    }

    public void AbortDialogue()
    {
        _dialogueInstance.stop(STOP_MODE.ALLOWFADEOUT);
    }

    [AOT.MonoPInvokeCallback(typeof(FMOD.Studio.EVENT_CALLBACK))]
    static FMOD.RESULT DialogueEventCallback(FMOD.Studio.EVENT_CALLBACK_TYPE type, IntPtr instancePtr,
        IntPtr parameterPtr)
    {
        FMOD.Studio.EventInstance instance = new FMOD.Studio.EventInstance(instancePtr);

        // Retrieve the user data
        IntPtr stringPtr;
        instance.getUserData(out stringPtr);

        // Get the string object
        GCHandle stringHandle = GCHandle.FromIntPtr(stringPtr);
        String key = stringHandle.Target as String;

        switch (type)
        {
            case FMOD.Studio.EVENT_CALLBACK_TYPE.CREATE_PROGRAMMER_SOUND:
            {
                FMOD.MODE soundMode = FMOD.MODE.LOOP_OFF | FMOD.MODE.CREATESAMPLE;
                var parameter =
                    (FMOD.Studio.PROGRAMMER_SOUND_PROPERTIES)Marshal.PtrToStructure(parameterPtr,
                        typeof(FMOD.Studio.PROGRAMMER_SOUND_PROPERTIES));

                if (key.Contains("."))
                {
#if UNITY_WEBGL
                    CreateSoundWebGL(Application.streamingAssetsPath + "/" + key, parameter, parameterPtr,
                        soundMode);
#else
                    CreateSoundWindows(Application.streamingAssetsPath + "/" + key, parameter, parameterPtr, soundMode);
#endif
                }
                else
                {
                    var keyResult = FMODUnity.RuntimeManager.StudioSystem.getSoundInfo(key, out SOUND_INFO dialogueSoundInfo);
                    if (keyResult != FMOD.RESULT.OK)
                    {
                        break;
                    }

                    string soundPath = Marshal.PtrToStringAnsi(dialogueSoundInfo.name_or_data);

#if UNITY_WEBGL
                    CreateSoundWebGL(soundPath, parameter, parameterPtr, soundMode | dialogueSoundInfo.mode,
                        dialogueSoundInfo.exinfo, dialogueSoundInfo.subsoundindex);
#else
                    CreateSoundWindows(soundPath, parameter, parameterPtr, soundMode | dialogueSoundInfo.mode,
                        dialogueSoundInfo.exinfo, dialogueSoundInfo.subsoundindex);
#endif
                }

                break;
            }
            case FMOD.Studio.EVENT_CALLBACK_TYPE.SOUND_STOPPED:
            {
                UnityEngine.Debug.Log("Sound stopped");
                break;
            }

            case FMOD.Studio.EVENT_CALLBACK_TYPE.DESTROY_PROGRAMMER_SOUND:
            {
                var parameter =
                    (FMOD.Studio.PROGRAMMER_SOUND_PROPERTIES)Marshal.PtrToStructure(parameterPtr,
                        typeof(FMOD.Studio.PROGRAMMER_SOUND_PROPERTIES));
                var sound = new FMOD.Sound(parameter.sound);

                sound.release();

                break; //todo dispatch onFinish callback event here (or similar)
            }
            case FMOD.Studio.EVENT_CALLBACK_TYPE.DESTROYED:
            {
                // Now the event has been destroyed, unpin the string memory so it can be garbage collected
                stringHandle.Free();

                break;
            }
        }

        return FMOD.RESULT.OK;
    }

    static void CreateSoundWebGL(string path, FMOD.Studio.PROGRAMMER_SOUND_PROPERTIES parameter, IntPtr parameterPtr,
        FMOD.MODE mode = FMOD.MODE.DEFAULT, FMOD.CREATESOUNDEXINFO exinfo = default, int subsoundIndex = -1)
    {
        UnityWebRequest www = UnityWebRequestMultimedia.GetAudioClip(path, AudioType.OGGVORBIS);
        www.SendWebRequest().completed += (operation) =>
        {
            if (www.result == UnityWebRequest.Result.Success)
            {
                byte[] soundData = www.downloadHandler.data;
                FMOD.Sound dialogueSound;
                var result = FMODUnity.RuntimeManager.CoreSystem.createSound(soundData,
                    mode | FMOD.MODE.CREATESAMPLE | FMOD.MODE.OPENMEMORY, ref exinfo, out dialogueSound);
                if (result == FMOD.RESULT.OK)
                {
                    parameter.sound = dialogueSound.handle;
                    parameter.subsoundIndex = subsoundIndex;
                    Marshal.StructureToPtr(parameter, parameterPtr, false);
                }
            }
            else
            {
                UnityEngine.Debug.LogError("Failed to load audio clip: " + www.error);
            }
        };
    }

    static void CreateSoundWindows(string path, FMOD.Studio.PROGRAMMER_SOUND_PROPERTIES parameter, IntPtr parameterPtr,
        FMOD.MODE mode = FMOD.MODE.DEFAULT, FMOD.CREATESOUNDEXINFO exinfo = default, int subsoundIndex = -1)
    {
        FMOD.Sound dialogueSound;
        var result = FMODUnity.RuntimeManager.CoreSystem.createSound(path,
            mode | FMOD.MODE.LOOP_OFF | FMOD.MODE.CREATESAMPLE | FMOD.MODE.NONBLOCKING, ref exinfo,
            out dialogueSound);
        if (result == FMOD.RESULT.OK)
        {
            parameter.sound = dialogueSound.handle;
            parameter.subsoundIndex = subsoundIndex;
            Marshal.StructureToPtr(parameter, parameterPtr, false);
        }
    }
}
2

Hi,

Thank you for bringing this to our attention!

I have managed to reproduce the issue you mentioned, unfortunately, I don’t have a workaround to it at that moment.

I have passed the details of this issue to our dev team for further investigation and will let you know if there’s any updates.

3

After discussing it with our dev team, I think you are on the right track to use WebRequest. And Just to double check, do you have any error logs pop out when the issue happened?

Due to the limitation of WebGL platform not supporting multi-threads, you might encounter an issue when trying to play a sound while the sound file hasn’t been fully loaded by the web server yet.

Also please note that using the async operation inside the programmer sound callback is not going to work, because the programmer sound callback is going to expect a valid and fully loaded sound when it returns, which means you might need to find another way to make sure no user input is affecting the files loading process from the web server.

For using programmer sound in WebGL platform, I have attached a modified example script here for you to take as reference:

using FMOD;
using System;
using System.Collections;
using System.Runtime.InteropServices;
using UnityEngine;
using UnityEngine.Networking;

public class TestProgrammerSound : MonoBehaviour
{
    public FMODUnity.EventReference EventName;

#if UNITY_EDITOR
    private string Url = Application.streamingAssetsPath + "/PianoSounds/1.mp3"; //replace with your file path here
#else
    private string Url = "StreamingAssets/PianoSounds/1.mp3"; //replace with your file path here
#endif

    public byte[] CachedAsset;

    [StructLayout(LayoutKind.Sequential)]
    class CallbackInfo
    {
        public Sound DialogueSound;
        public CREATESOUNDEXINFO Exinfo;
    }

    private CallbackInfo callbackInfo;
    private GCHandle callbackHandle;

    private FMOD.Studio.EVENT_CALLBACK dialogueCallback;

    private IEnumerator loadFromWeb(string assetPath)
    {
        byte[] loadWebResult;
        RESULT loadResult;

        UnityWebRequest www = UnityWebRequest.Get(assetPath);
        yield return www.SendWebRequest();
        CachedAsset = www.downloadHandler.data;
    }

    void Start()
    {
        dialogueCallback = new FMOD.Studio.EVENT_CALLBACK(DialogueEventCallback);
        StartCoroutine(loadFromWeb(Url));
    }

    void PlayDialogue()
    {
        callbackInfo = new CallbackInfo();

        callbackInfo.Exinfo = new CREATESOUNDEXINFO();

        callbackInfo.Exinfo.cbsize = Marshal.SizeOf(typeof(CREATESOUNDEXINFO));
        callbackInfo.Exinfo.length = (uint)CachedAsset.Length;
        callbackInfo.Exinfo.suggestedsoundtype = SOUND_TYPE.MPEG;
        callbackInfo.Exinfo.defaultfrequency = 44100;
        callbackInfo.Exinfo.format = SOUND_FORMAT.PCMFLOAT;

        RESULT soundResult = FMODUnity.RuntimeManager.CoreSystem.createSound(CachedAsset, MODE.CREATECOMPRESSEDSAMPLE | MODE.OPENMEMORY, ref callbackInfo.Exinfo, out callbackInfo.DialogueSound);
        if (soundResult != RESULT.OK)
        {
            UnityEngine.Debug.LogError("Failed to create sound: " + Error.String(soundResult));
            return;
        }

        var dialogueInstance = FMODUnity.RuntimeManager.CreateInstance(EventName);

        callbackHandle = GCHandle.Alloc(callbackInfo, GCHandleType.Pinned);

        dialogueInstance.setUserData(GCHandle.ToIntPtr(callbackHandle));
        dialogueInstance.setCallback(DialogueEventCallback);
        dialogueInstance.start();
        dialogueInstance.release();
    }

    [AOT.MonoPInvokeCallback(typeof(FMOD.Studio.EVENT_CALLBACK))]
    static RESULT DialogueEventCallback(FMOD.Studio.EVENT_CALLBACK_TYPE type, IntPtr instancePtr, IntPtr parameterPtr)
    {
        FMOD.Studio.EventInstance instance = new FMOD.Studio.EventInstance(instancePtr);

        // Retrieve the user data
        IntPtr callbackInfoPtr;
        RESULT result = instance.getUserData(out callbackInfoPtr);

        if (result != RESULT.OK)
        {
            UnityEngine.Debug.LogError("Timeline Callback error: " + result);
        }
        else if (callbackInfoPtr != IntPtr.Zero)
        {
            // Get the object to store beat and marker details
            GCHandle callbackHandle = GCHandle.FromIntPtr(callbackInfoPtr);
            CallbackInfo localCallbackInfo = (CallbackInfo)callbackHandle.Target;

            switch (type)
            {
                case FMOD.Studio.EVENT_CALLBACK_TYPE.CREATE_PROGRAMMER_SOUND:
                    {
                        var parameter = (FMOD.Studio.PROGRAMMER_SOUND_PROPERTIES)Marshal.PtrToStructure(parameterPtr, typeof(FMOD.Studio.PROGRAMMER_SOUND_PROPERTIES));

                        parameter.sound = localCallbackInfo.DialogueSound.handle;
                        parameter.subsoundIndex = -1;
                        Marshal.StructureToPtr(parameter, parameterPtr, false);
                        break;
                    }
                case FMOD.Studio.EVENT_CALLBACK_TYPE.DESTROY_PROGRAMMER_SOUND:
                    {
                        localCallbackInfo.DialogueSound.release();
                        break;
                    }
                case FMOD.Studio.EVENT_CALLBACK_TYPE.DESTROYED:
                    {
                        callbackHandle.Free();
                        break;
                    }
            }
        }

        return RESULT.OK;
    }

    void Update()
    {
        if (Input.GetKeyDown(KeyCode.Alpha1))
        {
            PlayDialogue();
        }
    }
}

Let me know if the issue persists, please don’t hesitate to ask more questions.