using Godot; using System; using System.Collections.Generic; using ETPreferences; using EyeTracking; /// /// Enum defining OSC paths for different eye tracking data /// public enum EyeTrackingPaths { EYES_CLOSED_AMOUNT, CENTER_PITCH_YAW, CENTER_PITCH_YAW_DIST, CENTER_VEC, CENTER_VEC_FULL, LEFT_RIGHT_PITCH_YAW, LEFT_RIGHT_VEC } /// /// Consumer for sending eye tracking data to VRChat via OSC /// public partial class VRChatConsumer : EyeTrackingConsumer { protected OscClient _oscClient; protected float _fovH; protected float _fovV; protected bool _debug; protected readonly Dictionary _pathMap; protected FrustumLoader _frustumLoader; // One Euro Filters for smoothing eye tracking data private OneEuroFilter _leftEyeXFilter; private OneEuroFilter _leftEyeYFilter; private OneEuroFilter _rightEyeXFilter; private OneEuroFilter _rightEyeYFilter; private EyeTrackingBuffer _dataBuffer; private bool _wasBlinking = false; private EyeTrackingData _lastStableFrame; private int _postBlinkHold = 0; // Lerp for eyelid motion around blink transitions private BlinkLerp _blinkLerp; // Filter configuration [Export] public float MinCutoff { get; set; } = 1.0f; // Minimum cutoff frequency [Export] public float Beta { get; set; } = ET_Preferences.filter_default; // Speed coefficient [Export] public float DCutoff { get; set; } = 1.0f; // Derivative cutoff frequency [Export] public int BlinkSmoothingFrames { get; set; } = 3; // frames to freeze after blink /// /// Initializes a new instance of the VRChatConsumer class /// /// The name of this consumer /// IP address to send OSC messages to /// Port to send OSC messages to public VRChatConsumer( string name = "VRChat Consumer") : base(name) { // Initialize frustum loader _frustumLoader = new FrustumLoader(); if (!_frustumLoader.LoadFrustumData()) { GD.PrintErr("Failed to load frustum data. Eye tracking angles may not be accurate."); throw new InvalidOperationException("Frustum data is required for VRChatConsumer to function properly."); } InitializeOSC(ET_Preferences.Instance.osc_ip, ET_Preferences.Instance.osc_port); _debug = false; // Initialize one euro filters InitializeFilters(); // Initialize path map _pathMap = new Dictionary { { EyeTrackingPaths.EYES_CLOSED_AMOUNT, "/tracking/eye/EyesClosedAmount" }, { EyeTrackingPaths.CENTER_PITCH_YAW, "/tracking/eye/CenterPitchYaw" }, { EyeTrackingPaths.CENTER_PITCH_YAW_DIST, "/tracking/eye/CenterPitchYawDist" }, { EyeTrackingPaths.CENTER_VEC, "/tracking/eye/CenterVec" }, { EyeTrackingPaths.CENTER_VEC_FULL, "/tracking/eye/CenterVecFull" }, { EyeTrackingPaths.LEFT_RIGHT_PITCH_YAW, "/tracking/eye/LeftRightPitchYaw" }, { EyeTrackingPaths.LEFT_RIGHT_VEC, "/tracking/eye/LeftRightVec" } }; // Initialize data buffer with custom analysis function _dataBuffer = new EyeTrackingBuffer( maxBufferSize: 40, minBufferSize: 10, analyzeChunkData: AnalyzeChunkData, processData: ProcessData, canProcess: () => true // VRChatConsumer doesn't have additional conditions ); // Initialize blink lerp _blinkLerp = new BlinkLerp(0.05f); } /// /// Initialize filter instances /// public void InitializeFilters() { // Set coefficient from user preference if (ET_Preferences.Instance != null && float.TryParse(ET_Preferences.Instance.smoothing_intensity, out float parsed)) { Beta = Math.Clamp(parsed, 0, 1); } _leftEyeXFilter = new OneEuroFilter(MinCutoff, Beta, DCutoff); _leftEyeYFilter = new OneEuroFilter(MinCutoff, Beta, DCutoff); _rightEyeXFilter = new OneEuroFilter(MinCutoff, Beta, DCutoff); _rightEyeYFilter = new OneEuroFilter(MinCutoff, Beta, DCutoff); } /// /// Initialize OSC client /// public async void InitializeOSC(string oscIp = "127.0.0.1", int oscPort = 9000) { if (_oscClient != null) { _oscClient.Close(); } _oscClient = new OscClient(oscIp, oscPort); GD.Print($"Initialized {Name} sending to {oscIp}:{oscPort}"); } /// /// Process the eye tracking data and send to VRChat /// /// Eye tracking data public override void OnEyeDataUpdate(EyeTrackingData data) { if (!Enabled) return; _dataBuffer.AddData(data); } private EyeTrackingData AnalyzeChunkData(List chunkData) { var frame = chunkData.Count > 0 ? chunkData[0] : new EyeTrackingData(); if (IsBlinkingFrame(frame)) { _postBlinkHold = BlinkSmoothingFrames; return frame; } if (_postBlinkHold > 0) { _postBlinkHold--; return _lastStableFrame; } _lastStableFrame = frame; return frame; } private bool IsBlinkingFrame(EyeTrackingData frame) { return frame.HasAnyFlagCondition(); } protected override void ProcessData(EyeTrackingData data) { try { bool filterDisabled = ET_Preferences.Instance != null && !ET_Preferences.Instance.smoothing_enabled; float currentTime = (float)Time.GetTicksMsec() / 1000.0f; BlinkMode blinkMode = EyeTrackingPreferences.GetBlinkMode(); // Compute base closed amount from openness values (0..1) float leftOpen = Math.Clamp(data.LeftEyeOpenness, 0f, 1f); float rightOpen = Math.Clamp(data.RightEyeOpenness, 0f, 1f); float avgOpen = (leftOpen + rightOpen) * 0.5f; float baseClosedAmount = 1.0f - avgOpen; baseClosedAmount = Math.Clamp(baseClosedAmount, 0f, 1f); // Apply blink behavior based on mode float eyesClosedAmount; bool treatAsBlinkingOrLerping = false; switch (blinkMode) { case BlinkMode.Blinking: { bool isCurrentFrameBlinking = IsBlinkingFrame(data); eyesClosedAmount = _blinkLerp.UpdateBlinkLerp(isCurrentFrameBlinking, currentTime, baseClosedAmount); treatAsBlinkingOrLerping = isCurrentFrameBlinking || _blinkLerp.IsLerping; break; } case BlinkMode.Winking: { // Map any eye closure to a shared channel; winks become full blinks here. bool anyEyeClosed = data.LeftEyeFlagCondition || data.RightEyeFlagCondition; eyesClosedAmount = _blinkLerp.UpdateBlinkLerp(anyEyeClosed, currentTime, baseClosedAmount); treatAsBlinkingOrLerping = anyEyeClosed || _blinkLerp.IsLerping; break; } case BlinkMode.None: default: { eyesClosedAmount = 0.0f; treatAsBlinkingOrLerping = false; break; } } // Filter and convert gaze to LeftRightPitchYaw float leftX, leftY; if (treatAsBlinkingOrLerping || filterDisabled) { leftX = data.LeftEyeX; leftY = data.LeftEyeY; } else { leftX = _leftEyeXFilter.Filter(data.LeftEyeX, currentTime); leftY = _leftEyeYFilter.Filter(data.LeftEyeY, currentTime); } float rightX = data.RightEyeX; float rightY = data.RightEyeY; bool singleEyeTracking = (rightX == 0f && rightY == 0f); if (singleEyeTracking) { rightX = leftX; rightY = leftY; } else { if (treatAsBlinkingOrLerping || filterDisabled) { // keep raw right eye values } else { rightX = _rightEyeXFilter.Filter(rightX, currentTime); rightY = _rightEyeYFilter.Filter(rightY, currentTime); } } float leftPitch = 0f, leftYaw = 0f, rightPitch = 0f, rightYaw = 0f; if (_frustumLoader.IsLoaded) { // Apply convergence logic to the gaze directions before converting to angles Vector2 leftGaze = new Vector2(leftX, leftY); Vector2 rightGaze = new Vector2(rightX, rightY); (Vector2 convergedLeft, Vector2 convergedRight) = EyeConvergence.ApplyConvergence(leftGaze, rightGaze); Vector2 leftAngles = _frustumLoader.ScreenPositionToViewAngles(convergedLeft, true); Vector2 rightAngles = _frustumLoader.ScreenPositionToViewAngles(convergedRight, false); leftYaw = leftAngles.X; rightYaw = rightAngles.X; // VRChat uses + down for pitch per docs, invert here leftPitch = -leftAngles.Y; rightPitch = -rightAngles.Y; } if (_oscClient != null) { // Eye-look _oscClient.SendMessage( _pathMap[EyeTrackingPaths.LEFT_RIGHT_PITCH_YAW], new float[] { leftPitch, leftYaw, rightPitch, rightYaw }); // Eyelids (single channel for both eyes) _oscClient.SendMessage( _pathMap[EyeTrackingPaths.EYES_CLOSED_AMOUNT], eyesClosedAmount); } } catch (Exception ex) { GD.PrintErr($"{Name}: Error sending OSC message: {ex.Message}"); GD.PrintErr(ex.StackTrace); } } }