using Godot;
using System;
using System.Collections.Generic;
using ETPreferences;
using EyeTracking;
///
/// Enum defining OSC paths for different eye tracking data
///
public enum EyeTrackingPaths
{
EYES_CLOSED_AMOUNT,
CENTER_PITCH_YAW,
CENTER_PITCH_YAW_DIST,
CENTER_VEC,
CENTER_VEC_FULL,
LEFT_RIGHT_PITCH_YAW,
LEFT_RIGHT_VEC
}
///
/// Consumer for sending eye tracking data to VRChat via OSC
///
public partial class VRChatConsumer : EyeTrackingConsumer
{
protected OscClient _oscClient;
protected float _fovH;
protected float _fovV;
protected bool _debug;
protected readonly Dictionary _pathMap;
protected FrustumLoader _frustumLoader;
// One Euro Filters for smoothing eye tracking data
private OneEuroFilter _leftEyeXFilter;
private OneEuroFilter _leftEyeYFilter;
private OneEuroFilter _rightEyeXFilter;
private OneEuroFilter _rightEyeYFilter;
private EyeTrackingBuffer _dataBuffer;
private bool _wasBlinking = false;
private EyeTrackingData _lastStableFrame;
private int _postBlinkHold = 0;
// Lerp for eyelid motion around blink transitions
private BlinkLerp _blinkLerp;
// Filter configuration
[Export]
public float MinCutoff { get; set; } = 1.0f; // Minimum cutoff frequency
[Export]
public float Beta { get; set; } = ET_Preferences.filter_default; // Speed coefficient
[Export]
public float DCutoff { get; set; } = 1.0f; // Derivative cutoff frequency
[Export]
public int BlinkSmoothingFrames { get; set; } = 3; // frames to freeze after blink
///
/// Initializes a new instance of the VRChatConsumer class
///
/// The name of this consumer
/// IP address to send OSC messages to
/// Port to send OSC messages to
public VRChatConsumer(
string name = "VRChat Consumer")
: base(name)
{
// Initialize frustum loader
_frustumLoader = new FrustumLoader();
if (!_frustumLoader.LoadFrustumData())
{
GD.PrintErr("Failed to load frustum data. Eye tracking angles may not be accurate.");
throw new InvalidOperationException("Frustum data is required for VRChatConsumer to function properly.");
}
InitializeOSC(ET_Preferences.Instance.osc_ip, ET_Preferences.Instance.osc_port);
_debug = false;
// Initialize one euro filters
InitializeFilters();
// Initialize path map
_pathMap = new Dictionary
{
{ EyeTrackingPaths.EYES_CLOSED_AMOUNT, "/tracking/eye/EyesClosedAmount" },
{ EyeTrackingPaths.CENTER_PITCH_YAW, "/tracking/eye/CenterPitchYaw" },
{ EyeTrackingPaths.CENTER_PITCH_YAW_DIST, "/tracking/eye/CenterPitchYawDist" },
{ EyeTrackingPaths.CENTER_VEC, "/tracking/eye/CenterVec" },
{ EyeTrackingPaths.CENTER_VEC_FULL, "/tracking/eye/CenterVecFull" },
{ EyeTrackingPaths.LEFT_RIGHT_PITCH_YAW, "/tracking/eye/LeftRightPitchYaw" },
{ EyeTrackingPaths.LEFT_RIGHT_VEC, "/tracking/eye/LeftRightVec" }
};
// Initialize data buffer with custom analysis function
_dataBuffer = new EyeTrackingBuffer(
maxBufferSize: 40,
minBufferSize: 10,
analyzeChunkData: AnalyzeChunkData,
processData: ProcessData,
canProcess: () => true // VRChatConsumer doesn't have additional conditions
);
// Initialize blink lerp
_blinkLerp = new BlinkLerp(0.05f);
}
///
/// Initialize filter instances
///
public void InitializeFilters()
{
// Set coefficient from user preference
if (ET_Preferences.Instance != null && float.TryParse(ET_Preferences.Instance.smoothing_intensity, out float parsed))
{
Beta = Math.Clamp(parsed, 0, 1);
}
_leftEyeXFilter = new OneEuroFilter(MinCutoff, Beta, DCutoff);
_leftEyeYFilter = new OneEuroFilter(MinCutoff, Beta, DCutoff);
_rightEyeXFilter = new OneEuroFilter(MinCutoff, Beta, DCutoff);
_rightEyeYFilter = new OneEuroFilter(MinCutoff, Beta, DCutoff);
}
///
/// Initialize OSC client
///
public async void InitializeOSC(string oscIp = "127.0.0.1", int oscPort = 9000)
{
if (_oscClient != null)
{
_oscClient.Close();
}
_oscClient = new OscClient(oscIp, oscPort);
GD.Print($"Initialized {Name} sending to {oscIp}:{oscPort}");
}
///
/// Process the eye tracking data and send to VRChat
///
/// Eye tracking data
public override void OnEyeDataUpdate(EyeTrackingData data)
{
if (!Enabled) return;
_dataBuffer.AddData(data);
}
private EyeTrackingData AnalyzeChunkData(List chunkData)
{
var frame = chunkData.Count > 0 ? chunkData[0] : new EyeTrackingData();
if (IsBlinkingFrame(frame)) { _postBlinkHold = BlinkSmoothingFrames; return frame; }
if (_postBlinkHold > 0) { _postBlinkHold--; return _lastStableFrame; }
_lastStableFrame = frame;
return frame;
}
private bool IsBlinkingFrame(EyeTrackingData frame)
{
return frame.HasAnyFlagCondition();
}
protected override void ProcessData(EyeTrackingData data)
{
try
{
bool filterDisabled = ET_Preferences.Instance != null && !ET_Preferences.Instance.smoothing_enabled;
float currentTime = (float)Time.GetTicksMsec() / 1000.0f;
BlinkMode blinkMode = EyeTrackingPreferences.GetBlinkMode();
// Compute base closed amount from openness values (0..1)
float leftOpen = Math.Clamp(data.LeftEyeOpenness, 0f, 1f);
float rightOpen = Math.Clamp(data.RightEyeOpenness, 0f, 1f);
float avgOpen = (leftOpen + rightOpen) * 0.5f;
float baseClosedAmount = 1.0f - avgOpen;
baseClosedAmount = Math.Clamp(baseClosedAmount, 0f, 1f);
// Apply blink behavior based on mode
float eyesClosedAmount;
bool treatAsBlinkingOrLerping = false;
switch (blinkMode)
{
case BlinkMode.Blinking:
{
bool isCurrentFrameBlinking = IsBlinkingFrame(data);
eyesClosedAmount = _blinkLerp.UpdateBlinkLerp(isCurrentFrameBlinking, currentTime, baseClosedAmount);
treatAsBlinkingOrLerping = isCurrentFrameBlinking || _blinkLerp.IsLerping;
break;
}
case BlinkMode.Winking:
{
// Map any eye closure to a shared channel; winks become full blinks here.
bool anyEyeClosed = data.LeftEyeFlagCondition || data.RightEyeFlagCondition;
eyesClosedAmount = _blinkLerp.UpdateBlinkLerp(anyEyeClosed, currentTime, baseClosedAmount);
treatAsBlinkingOrLerping = anyEyeClosed || _blinkLerp.IsLerping;
break;
}
case BlinkMode.None:
default:
{
eyesClosedAmount = 0.0f;
treatAsBlinkingOrLerping = false;
break;
}
}
// Filter and convert gaze to LeftRightPitchYaw
float leftX, leftY;
if (treatAsBlinkingOrLerping || filterDisabled)
{
leftX = data.LeftEyeX;
leftY = data.LeftEyeY;
}
else
{
leftX = _leftEyeXFilter.Filter(data.LeftEyeX, currentTime);
leftY = _leftEyeYFilter.Filter(data.LeftEyeY, currentTime);
}
float rightX = data.RightEyeX;
float rightY = data.RightEyeY;
bool singleEyeTracking = (rightX == 0f && rightY == 0f);
if (singleEyeTracking)
{
rightX = leftX;
rightY = leftY;
}
else
{
if (treatAsBlinkingOrLerping || filterDisabled)
{
// keep raw right eye values
}
else
{
rightX = _rightEyeXFilter.Filter(rightX, currentTime);
rightY = _rightEyeYFilter.Filter(rightY, currentTime);
}
}
float leftPitch = 0f, leftYaw = 0f, rightPitch = 0f, rightYaw = 0f;
if (_frustumLoader.IsLoaded)
{
// Apply convergence logic to the gaze directions before converting to angles
Vector2 leftGaze = new Vector2(leftX, leftY);
Vector2 rightGaze = new Vector2(rightX, rightY);
(Vector2 convergedLeft, Vector2 convergedRight) = EyeConvergence.ApplyConvergence(leftGaze, rightGaze);
Vector2 leftAngles = _frustumLoader.ScreenPositionToViewAngles(convergedLeft, true);
Vector2 rightAngles = _frustumLoader.ScreenPositionToViewAngles(convergedRight, false);
leftYaw = leftAngles.X;
rightYaw = rightAngles.X;
// VRChat uses + down for pitch per docs, invert here
leftPitch = -leftAngles.Y;
rightPitch = -rightAngles.Y;
}
if (_oscClient != null)
{
// Eye-look
_oscClient.SendMessage(
_pathMap[EyeTrackingPaths.LEFT_RIGHT_PITCH_YAW],
new float[] { leftPitch, leftYaw, rightPitch, rightYaw });
// Eyelids (single channel for both eyes)
_oscClient.SendMessage(
_pathMap[EyeTrackingPaths.EYES_CLOSED_AMOUNT],
eyesClosedAmount);
}
}
catch (Exception ex)
{
GD.PrintErr($"{Name}: Error sending OSC message: {ex.Message}");
GD.PrintErr(ex.StackTrace);
}
}
}