About Pixel Streaming

Unreal Engine
1

I created a build in Unreal 4 which includes Pixel Streaming Plugin.
When i try to connect it with a Chrome browser, the video output and the input is working properly.
But i could not hear any sound (Since all the sound in the game are using FMOD).
Is there anyone having a similar experience on this?

1 Like
2

Are you seeing any errors or warnings in the output log?

3

No. No errors in the log.
Do you have any details on how to config FMOD to make it work with Pixel Streaming?

4

After investigation, it looks like Pixel Streaming is not something the we can support at this time. At this point it looks like the Engine would need to be modified to stream the FMOD System, which the Engine currently does not know about.

I will raise this as a task to investigate further.

5

Great thanks.

6

Any news on this?

7

Nothing yet, I have bumped added a +1 to the task for this.

2 Likes
8

Hey Guys, I’m trying to make this work by modifying the AudioCapturer.cpp file of Pixel Streaming plugin.

So far, the function I’m inserting in Init() of AudioCapturer.cpp is:

if (IFMODStudioModule::IsAvailable())
	{
		FMOD_DSP_DESCRIPTION desc;
		memset(&desc, 0, sizeof(desc));
		strncpy(desc.name, "DSP Listener", sizeof(desc.name));
		desc.version = 0x00010000;
		desc.numinputbuffers = 1;
		desc.numoutputbuffers = 1;
		desc.read = OnNewFMODSubmixBuffer;
		desc.create = myDSPCreateCallback;
		desc.release = myDSPReleaseCallback;
		
		FMOD::Studio::System* StudioSystem = IFMODStudioModule::Get().GetStudioSystem(EFMODSystemContext::Runtime);
		
		if (StudioSystem)
		{
			StudioSystem->registerPlugin(&desc);
		}
	}

However, it seems that the read function is not even being executed. I implemented it like this:

FMOD_RESULT F_CALLBACK OnNewFMODSubmixBuffer(FMOD_DSP_STATE *dsp_state, float *AudioData, float *outbuffer, unsigned int NumSamples, int InNumChannels, int *outchannels)
{
	UE_LOG(LogAudioCapturer, Warning, TEXT("FMOD LISTENER"), NumSamples, InNumChannels, 48000);
	UE_LOG(LogAudioCapturer, Warning, TEXT("Captured FMOD %d samples, %dc, %dHz"), NumSamples, InNumChannels, 48000);

	if (!FAudioCapturer::instance)
	{
		return FMOD_ERR_NOTREADY;
	}

	FAudioCapturer* instance = FAudioCapturer::instance;

	// Only 48000hz supported for now
	/*if (InSampleRate != SampleRate)
	{
		// Only report the problem once
		if (!bFormatChecked)
		{
			bFormatChecked = true;
			UE_LOG(PixelStreaming, Error, TEXT("Audio samplerate needs to be 48000hz"));
		}
		return;
	}*/

	UE_LOG(LogAudioCapturer, Warning, TEXT("Captured FMOD 2 %d samples, %dc, %dHz"), NumSamples, InNumChannels, 48000);

	Audio::TSampleBuffer<float> Buffer(AudioData, NumSamples, InNumChannels, 48000);
	// Mix to stereo if required, since PixelStreaming only accepts stereo at the moment
	if (Buffer.GetNumChannels() != 1)
	{
		Buffer.MixBufferToChannels(1);
	}

	// Convert to signed PCM 16-bits
	instance->PCM16.Reset(Buffer.GetNumSamples());
	instance->PCM16.AddZeroed(Buffer.GetNumSamples());
	const float* Ptr = reinterpret_cast<const float*>(Buffer.GetData());
	for (int16& S : instance->PCM16)
	{
		int32 N = *Ptr >= 0 ? *Ptr * int32(MAX_int16) : *Ptr * (int32(MAX_int16) + 1);
		S = static_cast<int16>(FMath::Clamp(N, int32(MIN_int16), int32(MAX_int16)));
		Ptr++;
	}

	instance->RecordingBuffer.Append(reinterpret_cast<const uint8*>(instance->PCM16.GetData()), instance->PCM16.Num() * sizeof(instance->PCM16[0]));
	int BytesPer10Ms = (instance->SampleRate * instance->NumChannels * static_cast<int>(sizeof(uint16))) / 100;

	// Feed in 10ms chunks
	while (instance->RecordingBuffer.Num() >= BytesPer10Ms)
	{
		{
			FScopeLock Lock(&instance->DeviceBufferCS);
			if (instance->DeviceBuffer)
			{
				instance->DeviceBuffer->SetRecordedBuffer(instance->RecordingBuffer.GetData(), BytesPer10Ms / (sizeof(uint16) * instance->NumChannels));
				instance->DeviceBuffer->DeliverRecordedData();
				UE_LOG(LogAudioCapturer, VeryVerbose, TEXT("passed %d bytes"), BytesPer10Ms);
			}
		}

		instance->RecordingBuffer.RemoveAt(0, BytesPer10Ms, false);
	}
	return FMOD_OK;
};

/*
	Callback called when DSP is created.   This implementation creates a structure which is attached to the dsp state's 'plugindata' member.
*/
FMOD_RESULT F_CALLBACK myDSPCreateCallback(FMOD_DSP_STATE *dsp_state)
{
	unsigned int blocksize;
	FMOD_RESULT result;

	result = dsp_state->functions->getblocksize(dsp_state, &blocksize);
	//ERRCHECK(result);

	mydsp_data_t *data = (mydsp_data_t *)calloc(sizeof(mydsp_data_t), 1);
	if (!data)
	{
		return FMOD_ERR_MEMORY;
	}
	dsp_state->plugindata = data;
	data->volume_linear = 1.0f;
	data->length_samples = blocksize;

	data->buffer = (float *)malloc(blocksize * 8 * sizeof(float));      // *8 = maximum size allowing room for 7.1.   Could ask dsp_state->functions->getspeakermode for the right speakermode to get real speaker count.
	if (!data->buffer)
	{
		return FMOD_ERR_MEMORY;
	}

	return FMOD_OK;
}

/*
	Callback called when DSP is destroyed.   The memory allocated in the create callback can be freed here.
*/
FMOD_RESULT F_CALLBACK myDSPReleaseCallback(FMOD_DSP_STATE *dsp_state)
{
	if (dsp_state->plugindata)
	{
		mydsp_data_t *data = (mydsp_data_t *)dsp_state->plugindata;

		if (data->buffer)
		{
			free(data->buffer);
		}

		free(data);
	}

	return FMOD_OK;
}

All of this is silently failing. Am I missing something? Is this the right way to get the System reference? I’m trying for over a week to make it work, but had no success! hahaha