using Godot;
using System;
using System.Collections.Generic;
using System.IO.MemoryMappedFiles;
using System.Runtime.InteropServices;
using ETPreferences;
using EyeTracking;
namespace VRCFT
{
///
/// Memory-mapped data structure to share with C++ code
///
[StructLayout(LayoutKind.Sequential)]
public struct SharedGazeData
{
public float LeftEyeX;
public float LeftEyeY;
public float LeftEyeZ;
public float RightEyeX;
public float RightEyeY;
public float RightEyeZ;
public float CombinedX;
public float CombinedY;
public float CombinedZ;
public float Confidence;
public long Timestamp;
public int IsValid;
public float LeftEyeClosedAmount;
public float RightEyeClosedAmount;
}
///
/// Enum defining different data types for memory-mapped eye tracking data
///
public enum MemmapDataTypes
{
EYES_VECTORS,
EYES_ANGLES,
DEBUG_INFO
}
///
/// Consumer for sending eye tracking data to a memory-mapped file for VRCFT interop
///
public partial class VRCFTConsumer : EyeTrackingConsumer
{
private MemoryMappedFile _sharedMem;
private MemoryMappedViewAccessor _accessor;
private bool _running;
private bool _debug;
private const string SharedMemoryName = "VRCFTMemmapData";
protected FrustumLoader _frustumLoader;
// Buffer for eye tracking data - processes chunks of sequential data
private EyeTrackingBuffer _dataBuffer;
// One Euro Filters for smoothing eye tracking data
private OneEuroFilter _leftEyeXFilter;
private OneEuroFilter _leftEyeYFilter;
private OneEuroFilter _rightEyeXFilter;
private OneEuroFilter _rightEyeYFilter;
// Track previous frame state for blink transition detection
private bool _wasBlinking = false;
// Minimal state to freeze around blinks
private EyeTrackingData _lastStableFrame;
private int _postBlinkHold = 0;
// Eye openness lerping
private BlinkLerp _blinkLerp;
// Filter configuration
[Export]
public float MinCutoff { get; set; } = 1.0f; // Minimum cutoff frequency
[Export]
public float Beta { get; set; } = ET_Preferences.filter_default; // Speed coefficient
[Export]
public float DCutoff { get; set; } = 1.0f; // Derivative cutoff frequency
[Export]
public int BlinkSmoothingFrames { get; set; } = 3; // Number of frames to check before/after blink for smoothing
///
/// Initializes a new instance of the VRCFTConsumer class
///
/// The name of this consumer
public VRCFTConsumer(string name = "VRCFT Consumer") : base(name)
{
// Initialize frustum loader
_frustumLoader = new FrustumLoader();
if (!_frustumLoader.LoadFrustumData())
{
GD.PrintErr("Failed to load frustum data. Eye tracking will not function without frustum data.");
throw new InvalidOperationException("Frustum data is required for VRCFTConsumer to function properly.");
}
// Initialize data buffer with custom analysis function
_dataBuffer = new EyeTrackingBuffer(
maxBufferSize: 40,
minBufferSize: 10,
analyzeChunkData: AnalyzeChunkData,
processData: ProcessData,
canProcess: () => _running && _accessor != null
);
InitializeFilters();
InitializeMemmap();
// Initialize blink lerp
_blinkLerp = new BlinkLerp(0.05f);
}
///
/// Initialize filter instances
///
public void InitializeFilters()
{
// Set coefficient from user preference
if (ET_Preferences.Instance != null && float.TryParse(ET_Preferences.Instance.smoothing_intensity, out float parsed))
{
Beta = Math.Clamp(parsed, 0, 1);
}
_leftEyeXFilter = new OneEuroFilter(MinCutoff, Beta, DCutoff);
_leftEyeYFilter = new OneEuroFilter(MinCutoff, Beta, DCutoff);
_rightEyeXFilter = new OneEuroFilter(MinCutoff, Beta, DCutoff);
_rightEyeYFilter = new OneEuroFilter(MinCutoff, Beta, DCutoff);
}
// Dispose of resources
private void Cleanup()
{
if (_accessor != null)
{
try
{
// Set data as invalid when stopping
var data = new SharedGazeData { IsValid = 0 };
_accessor.Write(0, ref data);
_accessor.Dispose();
_accessor = null;
GD.Print($"{Name}: View accessor closed");
}
catch (Exception ex)
{
GD.PrintErr($"{Name}: Error closing view accessor: {ex.Message}");
GD.PrintErr(ex.StackTrace);
}
}
if (_sharedMem != null)
{
try
{
_sharedMem.Dispose();
_sharedMem = null;
GD.Print($"{Name}: Shared memory closed");
}
catch (Exception ex)
{
GD.PrintErr($"{Name}: Error closing shared memory: {ex.Message}");
GD.PrintErr(ex.StackTrace);
}
}
}
///
/// Initialize the memory-mapped file
///
private void InitializeMemmap()
{
// Any existing should be disposed
Cleanup();
try
{
// Create a memory-mapped file of the size of SharedGazeData
_sharedMem = MemoryMappedFile.CreateOrOpen(
SharedMemoryName,
Marshal.SizeOf(),
MemoryMappedFileAccess.ReadWrite);
// Create a view accessor for the memory-mapped file
_accessor = _sharedMem.CreateViewAccessor(0, Marshal.SizeOf());
// Initialize with zeros and mark as invalid
var data = new SharedGazeData { IsValid = 0 };
_accessor.Write(0, ref data);
GD.Print($"{Name}: Successfully created shared memory");
_running = true;
}
catch (Exception ex)
{
GD.PrintErr($"{Name}: Failed to create shared memory: {ex.Message}");
GD.PrintErr(ex.StackTrace);
_running = false;
}
}
///
/// Process eye tracking data and send to the shared memory
///
/// Eye tracking data
public override void OnEyeDataUpdate(EyeTrackingData data)
{
if (Enabled)
{
_dataBuffer.AddData(data);
}
}
private EyeTrackingData AnalyzeChunkData(List chunkData)
{
var frame = chunkData.Count > 0 ? chunkData[0] : new EyeTrackingData();
if (IsBlinkingFrame(frame)) { _postBlinkHold = BlinkSmoothingFrames; return frame; }
if (_postBlinkHold > 0) { _postBlinkHold--; return _lastStableFrame; }
_lastStableFrame = frame;
return frame;
}
private bool IsBlinkingFrame(EyeTrackingData frame)
{
return frame.HasAnyFlagCondition();
}
///
/// Process eye tracking data and update shared memory
///
/// Eye tracking data
protected override void ProcessData(EyeTrackingData data)
{
if (!_running || _accessor == null)
return;
if (!_frustumLoader.IsLoaded)
{
GD.PrintErr($"{Name}: Frustum data is not loaded. Cannot compute gaze without frustum data.");
return;
}
try
{
// User preference for smoothing filter
bool filterDisabled = ET_Preferences.Instance != null && !ET_Preferences.Instance.smoothing_enabled;
BlinkMode blinkMode = EyeTrackingPreferences.GetBlinkMode();
float currentTime = (float)Time.GetTicksMsec() / 1000.0f;
// Base closed amounts from openness values
float leftBaseClosedAmount = 1.0f - Math.Clamp(data.LeftEyeOpenness, 0f, 1f);
float rightBaseClosedAmount = 1.0f - Math.Clamp(data.RightEyeOpenness, 0f, 1f);
float leftX, leftY, rightX, rightY;
float leftEyeClosedAmount = 0.0f;
float rightEyeClosedAmount = 0.0f;
bool treatAsBlinkingOrLerping = false;
switch (blinkMode)
{
case BlinkMode.Blinking:
{
bool isCurrentFrameBlinking = IsBlinkingFrame(data);
(leftEyeClosedAmount, rightEyeClosedAmount) = _blinkLerp.UpdateBlinkLerp(
isCurrentFrameBlinking,
currentTime,
leftBaseClosedAmount,
rightBaseClosedAmount);
treatAsBlinkingOrLerping = isCurrentFrameBlinking || _blinkLerp.IsLerping;
break;
}
case BlinkMode.Winking:
{
bool isLeftBlinking = data.LeftEyeFlagCondition;
bool isRightBlinking = data.RightEyeFlagCondition;
(leftEyeClosedAmount, rightEyeClosedAmount) = _blinkLerp.UpdateBlinkLerpPerEye(
isLeftBlinking,
isRightBlinking,
currentTime,
leftBaseClosedAmount,
rightBaseClosedAmount
);
treatAsBlinkingOrLerping = isLeftBlinking || isRightBlinking || _blinkLerp.IsLerping;
// If an eye is winking, set amounts for both to make it clear
bool leftWinking = data.LeftEyeFlagCondition && !data.RightEyeFlagCondition;
bool rightWinking = !data.LeftEyeFlagCondition && data.RightEyeFlagCondition;
if (leftWinking)
{
leftEyeClosedAmount = 1.0f;
rightEyeClosedAmount = 0.0f;
}
else if (rightWinking)
{
rightEyeClosedAmount = 1.0f;
leftEyeClosedAmount = 0.0f;
}
break;
}
case BlinkMode.None:
default:
{
leftEyeClosedAmount = 0.0f;
rightEyeClosedAmount = 0.0f;
treatAsBlinkingOrLerping = false;
break;
}
}
// Decide whether to use raw or filtered gaze based on blink mode and state
if (treatAsBlinkingOrLerping || filterDisabled)
{
// For blink/wink frames, use raw values (don't filter)
leftX = data.LeftEyeX;
leftY = data.LeftEyeY;
rightX = data.RightEyeX;
rightY = data.RightEyeY;
}
else
{
// Apply one euro filter to smooth eye positions
leftX = _leftEyeXFilter.Filter(data.LeftEyeX, currentTime);
leftY = _leftEyeYFilter.Filter(data.LeftEyeY, currentTime);
rightX = _rightEyeXFilter.Filter(data.RightEyeX, currentTime);
rightY = _rightEyeYFilter.Filter(data.RightEyeY, currentTime);
}
// When only tracking with left eye, use left eye data for both eyes
bool singleEyeTracking = (data.RightEyeX == 0 && data.RightEyeY == 0);
if (singleEyeTracking)
{
rightX = leftX;
rightY = leftY;
}
if (_debug)
{
GD.Print($"Raw - Left: ({data.LeftEyeX:F3}, {data.LeftEyeY:F3}), Right: ({data.RightEyeX:F3}, {data.RightEyeY:F3})");
GD.Print($"Filtered - Left: ({leftX:F3}, {leftY:F3}), Right: ({rightX:F3}, {rightY:F3})");
GD.Print($"Flag Conditions - Left: {data.LeftEyeFlagCondition} (prob: {data.LeftEyeFlagProbability:F3}), Right: {data.RightEyeFlagCondition} (prob: {data.RightEyeFlagProbability:F3})");
}
// Convert eye positions to pitch and yaw using frustum loader
// Convert normalized screen positions (0-1 range) to view angles
Vector2 leftAngles = _frustumLoader.ScreenPositionToViewAngles(new Vector2(leftX, leftY), true);
Vector2 rightAngles = _frustumLoader.ScreenPositionToViewAngles(new Vector2(rightX, rightY), false);
// Extract pitch (vertical) and yaw (horizontal) angles
float leftYaw = leftAngles.X;
float leftPitch = leftAngles.Y;
float rightYaw = rightAngles.X;
float rightPitch = rightAngles.Y;
if (_debug)
{
GD.Print($"Left eye - Pitch: {leftPitch}, Yaw: {leftYaw}");
GD.Print($"Right eye - Pitch: {rightPitch}, Yaw: {rightYaw}");
}
// Apply convergence logic to the gaze directions before converting to 3D vectors
Vector2 leftGaze = new Vector2(leftYaw, leftPitch);
Vector2 rightGaze = new Vector2(rightYaw, rightPitch);
(Vector2 convergedLeft, Vector2 convergedRight) = EyeConvergence.ApplyConvergence(leftGaze, rightGaze);
// Convert converged angles to vectors
Vector3 leftVector = AnglesToVector(convergedLeft.Y, convergedLeft.X);
Vector3 rightVector = AnglesToVector(convergedRight.Y, convergedRight.X);
// Calculate combined gaze direction (average of both eyes)
Vector3 combinedVector = new Vector3(
(leftVector.X + rightVector.X) / 2,
(leftVector.Y + rightVector.Y) / 2,
(leftVector.Z + rightVector.Z) / 2).Normalized();
// Calculate confidence
float confidence = data.Confidence;
// Create shared data structure
var sharedData = new SharedGazeData
{
LeftEyeX = leftVector.X,
LeftEyeY = leftVector.Y,
LeftEyeZ = leftVector.Z,
RightEyeX = rightVector.X,
RightEyeY = rightVector.Y,
RightEyeZ = rightVector.Z,
CombinedX = combinedVector.X,
CombinedY = combinedVector.Y,
CombinedZ = combinedVector.Z,
Confidence = confidence,
Timestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
IsValid = 1,
LeftEyeClosedAmount = leftEyeClosedAmount,
RightEyeClosedAmount = rightEyeClosedAmount
};
// Write to shared memory
_accessor.Write(0, ref sharedData);
}
catch (Exception ex)
{
GD.PrintErr($"{Name}: Error updating shared memory: {ex.Message}");
GD.PrintErr(ex.StackTrace);
}
}
///
/// Convert pitch and yaw angles to a 3D direction vector
///
/// Pitch angle in degrees
/// Yaw angle in degrees
/// Direction vector
private Vector3 AnglesToVector(float pitch, float yaw)
{
// Convert degrees to radians
float pitchRad = (float)(pitch * Math.PI / 180.0);
float yawRad = (float)(yaw * Math.PI / 180.0);
// Calculate direction vector
float x = (float)(Math.Sin(yawRad) * Math.Cos(pitchRad));
float y = (float)Math.Sin(pitchRad);
float z = (float)(Math.Cos(yawRad) * Math.Cos(pitchRad));
return new Vector3(x, y, -z); // Note: negating z as forward is -z in many 3D engines
}
public override void Enable()
{
base.Enable();
// Create resources
InitializeMemmap();
}
///
/// Stop the consumer and clean up resources
///
public override void Disable()
{
base.Disable(); // Call the base class Stop method first
// Clear the data buffer
_dataBuffer.Clear();
// Reset blink states
_wasBlinking = false;
_blinkLerp.Reset();
Cleanup();
_running = false;
}
}
}