Can we use fmod with videoplayer

Unity
6

Hi, just wanted to share that I also had a lot of problems with the FMOD video playback sample https://www.fmod.com/docs/2.03/unity/examples-video-playback.html, and have attached a rewrite that fixes these issues:

  • Works on macOS (AudioSampleProvider callback doesn’t get called on macOS, presumably a Unity bug, so uses polling in update instead), also tested on Windows.
  • Writes to end of write cursor instead of to last read cursor, fixes stuttering playback
  • Pauses playback until buffer is filled, fixes stuttering playback
  • Buffers through one NativeArray instead of allocating/resizing/copying through multiple C# lists and arrays
  • Removed dynamic resampling based on latency measurement, should not be necessary
using Unity.Collections;
using UnityEditor;
using UnityEngine;
using UnityEngine.Experimental.Audio;
using UnityEngine.Experimental.Video;
using UnityEngine.Video;
using System.Runtime.InteropServices;
using System.Collections.Generic;
using System;
using UnityEngine.UIElements;
using Unity.Collections.LowLevel.Unsafe;

/// <summary>
/// Route video audio through FMOD; based on
/// https://www.fmod.com/docs/2.03/unity/examples-video-playback.html
/// 
/// - Poll in Update instead of using SampleFramesAvailable event, as the event is not invoked on macOS
/// - Avoid buffering through C# list, rely on AudioSampleProvider's internal buffer and lock/unlock FMOD sound directly
///   with one intermediate temp NativeArray buffer
/// - Pause channel until buffer is filled
/// - Write samples after write cursor instead of after last frame's read cursor to fix stuttering
/// </summary>
public class VideoPlayerAudioPlayer : MonoBehaviour
{
    private const int LATENCY_MS = 50;

    private VideoPlayer mVideoPlayer;
    private AudioSampleProvider mProvider;

    private FMOD.CREATESOUNDEXINFO mExinfo;
    private FMOD.Channel mChannel;
    private FMOD.Sound mSound;
    private uint mWriteCursor;

    void Start()
    {
#if UNITY_EDITOR
        EditorApplication.pauseStateChanged += EditorStateChange;
#endif
    }

#if UNITY_EDITOR
    private void EditorStateChange(PauseState state)
    {
        if (mChannel.hasHandle())
        {
            mChannel.setPaused(state == PauseState.Paused);
        }
    }
#endif

    private void OnDestroy()
    {
        mChannel.stop();
        mSound.release();

#if UNITY_EDITOR
        EditorApplication.pauseStateChanged -= EditorStateChange;
#endif
    }

    public void Setup(VideoPlayer videoPlayer)
    {
        this.mVideoPlayer = videoPlayer;
    }

    public void OnLoopPointReached(VideoPlayer vp)
    {
        if (!vp.isLooping)
        {
            mChannel.setPaused(true);
        }
    }

    public void OnPrepared(VideoPlayer vp)
    {
        mProvider = vp.GetAudioSampleProvider(0);
        int sampleRate = (int)(mProvider.sampleRate * mVideoPlayer.playbackSpeed);
        uint targetLatencySamples = (uint)(sampleRate * LATENCY_MS) / 1000;

        mExinfo.cbsize = Marshal.SizeOf(typeof(FMOD.CREATESOUNDEXINFO));
        mExinfo.numchannels = mProvider.channelCount;
        mExinfo.defaultfrequency = sampleRate;
        mExinfo.length = targetLatencySamples * (uint)mExinfo.numchannels * sizeof(float);
        mExinfo.format = FMOD.SOUND_FORMAT.PCMFLOAT;
        FMODUnity.RuntimeManager.CoreSystem.createSound("", FMOD.MODE.LOOP_NORMAL | FMOD.MODE.OPENUSER, ref mExinfo, out mSound);

        FMOD.ChannelGroup channelGroup;
        FMODUnity.RuntimeManager.CoreSystem.getMasterChannelGroup(out channelGroup);
        FMODUnity.RuntimeManager.CoreSystem.playSound(mSound, channelGroup, true, out mChannel);
    }

    private void Update()
    {
        if (!mVideoPlayer || !mVideoPlayer.isPrepared)
            return;

        uint availableSamples = mProvider.availableSampleFrameCount;
        if (availableSamples > 0)
        {
            mChannel.getPosition(out uint readCursorLoop, FMOD.TIMEUNIT.PCMBYTES);
            uint writeCursorLoop = mWriteCursor % mExinfo.length;
            uint maxWriteBytes = readCursorLoop - writeCursorLoop;
            if (readCursorLoop < writeCursorLoop || mWriteCursor == 0)
                maxWriteBytes += mExinfo.length;

            if (maxWriteBytes > 0)
            {
                using (var buffer = new NativeArray<byte>((int)maxWriteBytes, Allocator.Temp))
                {
                    uint samples = mProvider.ConsumeSampleFrames(buffer.Reinterpret<float>(1));
                    uint bytes = samples * (uint)mExinfo.numchannels * sizeof(float);
                    var res = mSound.@lock(writeCursorLoop, bytes, out IntPtr ptr1, out IntPtr ptr2, out uint lenBytes1, out uint lenBytes2);
                    if (res != FMOD.RESULT.OK) Debug.LogError(res);

                    unsafe
                    {
                        byte* src = (byte*)buffer.GetUnsafeReadOnlyPtr();
                        UnsafeUtility.MemCpy((byte*)ptr1, src, lenBytes1);
                        UnsafeUtility.MemCpy((byte*)ptr2, src + lenBytes1, lenBytes2);
                    }

                    res = mSound.unlock(ptr1, ptr2, lenBytes1, lenBytes2);
                    if (res != FMOD.RESULT.OK) Debug.LogError(res);

                    mWriteCursor += bytes;
                }
            }
        }

        if (mWriteCursor >= mExinfo.length)
            mChannel.setPaused(false);  

#if UNITY_EDITOR
        if (mChannel.hasHandle() && mChannel.getChannelGroup(out var channelGroup) == FMOD.RESULT.OK)
            channelGroup.setMute(EditorUtility.audioMasterMute);
#endif
    }
}