using Godot; using System; using System.Threading; using System.Threading.Tasks; using OpenCvSharp; using ETPreferences; using HidSharp; using System.Linq; using System.Collections.Generic; /// /// Processes eye tracking data. /// public class EyeDataProcessor { /// /// Maximum age of eye data in seconds before it's considered stale. /// public float MaxEyeDataAge { get; set; } = 0.1f; /// /// Validates if eye data is fresh enough to use with the current frame. /// /// The eye data to validate. /// The timestamp of the current frame. /// True if the eye data is valid for the current frame, false otherwise. public bool ValidateEyeData(EyeData eyeData, float frameTimestamp) { if (eyeData == null) return false; // Check if the eye data is too old float age = frameTimestamp - eyeData.Timestamp; return age >= 0 && age <= MaxEyeDataAge; } } /// /// Simple visualization for eye tracking data. /// public class Visualizer { private bool _initialized = false; /// /// Initializes the visualizer. /// public void InitRerun() { _initialized = true; GD.Print("Visualizer initialized"); } /// /// Updates the visualization with the latest frame and eye data. /// /// The current frame number. /// The current frame as an OpenCV Mat. /// The current eye position data. public void UpdateVisualization(int frameCount, Mat frame, EyeData eyeData) { if (!_initialized) { GD.PrintErr("Visualizer not initialized. Call InitRerun() first."); return; } // Draw eye positions on the frame for visualization DrawEyePositions(frame, eyeData); // In a real implementation, this would update a visualization window or stream GD.Print($"Visualizing frame {frameCount} with eye positions: " + $"L({eyeData.LeftPos.X:F2}, {eyeData.LeftPos.Y:F2}), " + $"R({eyeData.RightPos.X:F2}, {eyeData.RightPos.Y:F2})"); } /// /// Draws eye positions on the frame for visualization. /// /// The frame to draw on. /// The eye position data. private void DrawEyePositions(Mat frame, EyeData eyeData) { // Convert normalized coordinates [0,1] to pixel coordinates int leftX = (int)(eyeData.LeftPos.X * frame.Width); int leftY = (int)(eyeData.LeftPos.Y * frame.Height); int rightX = (int)(eyeData.RightPos.X * frame.Width); int rightY = (int)(eyeData.RightPos.Y * frame.Height); // Draw circles at eye positions using OpenCV Cv2.Circle(frame, new Point(leftX, leftY), 10, new Scalar(0, 0, 255), -1); // Red circle for left eye Cv2.Circle(frame, new Point(rightX, rightY), 10, new Scalar(255, 0, 0), -1); // Blue circle for right eye } } /// /// Extended EyeData class with timestamp. /// public partial class EyeData { /// /// Timestamp when the eye data was captured. /// public float Timestamp { get; set; } } /// /// Manages the acquisition of eye tracking data and camera frames. /// public partial class AcquisitionManager : Node3D { [Export] public int FrameRate { get; set; } = 30; [Export] public Camera3D VrCamera { get; set; } [Export] public XROrigin3D XrOrigin { get; set; } [Export] public Node3D Target { get; set; } [Export] public TestAcquisition AcquisitionSource { get; set; } [Export] public float Distance { get; set; } = 1.5f; public Vector2 TargetInScreenSpace { get; private set; } public int DemoBlinkNotation { get; set; } = 0; public ET_Preferences etSettings { get { return ET_Preferences.Instance; } } public int OriginalHeadsetBrightness { get; set; } = -1; public bool AppQuitting { get; set; } private float _frameInterval; private CameraManager _cameraManager; private bool _recording = false; private bool _paused = false; private Thread _acquisitionThread; private CancellationTokenSource _cancellationTokenSource; private DataRecorder _dataRecorder; private Visualizer _visualizer; private EyeDataProcessor _eyeProcessor; private int _frameCount = 0; private EyeData _lastEyeData = null; private Projection _leftProjection; private Projection _rightProjection; private Transform3D _leftEyeTransform; private Transform3D _rightEyeTransform; private bool _frustumRecorded = false; private HidDevice hidDevice = null; // Target device VID and PID const int VID = 0x35bd; const int PID = 0x0101; private const uint CONFIG_DATA_LENGTH = 512; private List<(byte tag, string data)> headsetConfig = new List<(byte tag, string data)>(); private byte[] raw_config = new byte[512]; private byte waiting_on_block; private bool requesting_config; /// /// Called when the node enters the scene tree for the first time. /// public override void _Ready() { _frameInterval = 1.0f / FrameRate; // Initialize components _dataRecorder = new DataRecorder(); _visualizer = new Visualizer(); _eyeProcessor = new EyeDataProcessor(); // Create camera manager _cameraManager = new CameraManager(); _cameraManager.DesiredWidth = 800; _cameraManager.DesiredHeight = 400; AddChild(_cameraManager); // Fetch preferences EyeTrackingPreferences.ReadSettingsFile(); // Ensure we have the required references if (VrCamera == null) { GD.PrintErr("VR Camera reference is missing. Please assign it in the Inspector."); } if (XrOrigin == null) { GD.PrintErr("XR Origin reference is missing. Please assign it in the Inspector."); } if (Target == null) { GD.PrintErr("Target reference is missing. Please assign it in the Inspector."); } OpenXRManager.EnableOpenXR(this); FetchHidDevice(); Task.Run(async () => { HidLoop(); // Attempt to fetch the headset brightness to restore after enrollment OriginalHeadsetBrightness = GetCurrentBrightness().Result; GD.Print("Found headset brightness: " + OriginalHeadsetBrightness); }); } /// /// Called every frame to update the rig position. /// public override void _Process(double delta) { if (_recording) { var xrInterface = XRServer.FindInterface("OpenXR"); if (xrInterface != null) { Vector2 viewport = xrInterface.GetRenderTargetSize(); float aspectRatio = viewport.X / viewport.Y; _leftProjection = xrInterface.GetProjectionForView(0, aspectRatio, VrCamera.Near, VrCamera.Far); _rightProjection = xrInterface.GetProjectionForView(1, aspectRatio, VrCamera.Near, VrCamera.Far); // Update eye transforms in the main thread since they depend on current XR state _leftEyeTransform = xrInterface.GetTransformForView(0, XrOrigin.GlobalTransform); _rightEyeTransform = xrInterface.GetTransformForView(1, XrOrigin.GlobalTransform); if (!_frustumRecorded) { _dataRecorder.RecordFrustum(_leftProjection, _rightProjection, VrCamera.Near, VrCamera.Far); _frustumRecorded = true; } } else { throw new Exception("OpenXR interface not found"); } RepositionRig(); UpdateScreenSpace(); } } /// /// Repositions the rig based on the VR camera's forward direction. /// private void RepositionRig() { if (Target == null || VrCamera == null) { GD.PrintErr("Target or VR Camera is not assigned."); return; } // 1. Get the camera's current forward direction (world space) Vector3 cameraForward = -VrCamera.GlobalTransform.Basis.Z; // 2. We only want to use the horizontal component of the camera's forward direction // This prevents vertical "headlocking" while maintaining the correct Z distance Vector3 cameraForwardHorizontal = new Vector3(0, 0, cameraForward.Z).Normalized(); // 3. Calculate the desired position Vector3 cameraLocalPos = VrCamera.Position; // local offset Vector3 desiredCameraWorldPos = Target.GlobalPosition - cameraForwardHorizontal * Distance; Vector3 newRigPosition = desiredCameraWorldPos - cameraLocalPos; // 4. Move the XR Origin to that position GlobalPosition = newRigPosition; } /// /// Sets up the camera for acquisition. /// private async Task SetupCamera() { try { bool success = await _cameraManager.InitializeCamera(); if (!success) { GD.PrintErr("Failed to initialize camera"); return false; } GD.Print("Camera setup complete"); return true; } catch (Exception ex) { GD.PrintErr($"Error setting up camera: {ex.Message}"); GD.PrintErr(ex.StackTrace); return false; } } /// /// Gets eye tracking data directly from the VR system. /// /// The latest eye data, or null if not available. private EyeData GetEyeTrackingData() { try { if (VrCamera == null || XrOrigin == null || Target == null) { return null; } var xrInterface = XRServer.FindInterface("OpenXR"); if (xrInterface == null) { GD.PrintErr("OpenXR interface not found"); return null; } Vector2 viewport = xrInterface.GetRenderTargetSize(); float aspectRatio = viewport.X / viewport.Y; // Use the stored transforms that were updated in _Process Vector2 leftViewportPos = ProjectPointWithTransform(Target.GlobalTransform.Origin, _leftEyeTransform, _leftProjection); Vector2 rightViewportPos = ProjectPointWithTransform(Target.GlobalTransform.Origin, _rightEyeTransform, _rightProjection); // Create eye data with current timestamp var eyeData = new EyeData(leftViewportPos, rightViewportPos); eyeData.Timestamp = (float)Time.GetTicksMsec() / 1000.0f; // Set eye openness eyeData.LeftOpenness = DemoBlinkNotation; eyeData.RightOpenness = DemoBlinkNotation; return eyeData; } catch (Exception ex) { GD.PrintErr($"Error getting eye tracking data: {ex.Message}"); return null; } } /// /// Projects a world point using a custom camera transform and projection matrix. /// /// The point in world space to project. /// The camera transform. /// The projection matrix. /// The projected point in viewport coordinates (range [0,1]). private Vector2 ProjectPointWithTransform(Vector3 worldPoint, Transform3D cameraTransform, Projection projection) { // Get the view matrix (inverse of camera transform) Transform3D viewMatrix = cameraTransform.Inverse(); // Transform the world point to view space Vector3 pointInView = viewMatrix * worldPoint; // Create a Vector4 for the projection Vector4 vec = new Vector4(pointInView.X, pointInView.Y, pointInView.Z, 1.0f); // Transform to clip space Vector4 clipSpace = projection * vec; // Convert to normalized device coordinates (NDC) Vector3 ndc; if (clipSpace.W != 0) { ndc = new Vector3(clipSpace.X, clipSpace.Y, clipSpace.Z) / clipSpace.W; } else { ndc = new Vector3(clipSpace.X, clipSpace.Y, clipSpace.Z); } // Convert from NDC (range [-1,1]) to viewport coordinates (range [0,1]) float viewportX = (ndc.X * 0.5f) + 0.5f; float viewportY = (ndc.Y * 0.5f) + 0.5f; return new Vector2(viewportX, viewportY); } /// /// Processes a single frame of data. /// /// Token to monitor for cancellation requests. /// True if processing should continue, false otherwise. private bool ProcessFrame(CancellationToken cancellationToken) { float loopStart = (float)Time.GetTicksMsec() / 1000.0f; try { // Check for cancellation if (cancellationToken.IsCancellationRequested) return false; // Capture frame Mat frame = _cameraManager.GetCameraFrame(); if (frame == null || frame.Empty()) { GD.PrintErr("Failed to capture frame"); return true; // Continue trying } float frameTimestamp = (float)Time.GetTicksMsec() / 1000.0f; _frameCount++; // Get latest eye data EyeData eyeData = GetEyeTrackingData(); if (eyeData != null) { _lastEyeData = eyeData; GD.Print($"Updated eye data: L({_lastEyeData.LeftPos.X:F2}, {_lastEyeData.LeftPos.Y:F2}), " + $"R({_lastEyeData.RightPos.X:F2}, {_lastEyeData.RightPos.Y:F2})"); } // Process synchronized data if (_lastEyeData != null && _eyeProcessor.ValidateEyeData(_lastEyeData, frameTimestamp)) { GD.Print($"Synchronized frame {_frameCount} at {frameTimestamp:F3} " + $"with eye data at {_lastEyeData.Timestamp:F3}"); // Record data _dataRecorder.RecordFrame(frame, _frameCount); _dataRecorder.RecordEyeData(_frameCount, frameTimestamp, _lastEyeData); // Update visualization _visualizer.UpdateVisualization(_frameCount, frame, _lastEyeData); } else { GD.Print($"No fresh eye data for frame {_frameCount} at {frameTimestamp:F3}"); } // Maintain frame rate float elapsed = ((float)Time.GetTicksMsec() / 1000.0f) - loopStart; int sleepMs = (int)Math.Max(0, (_frameInterval - elapsed) * 1000); if (sleepMs > 0) { Thread.Sleep(sleepMs); } return true; } catch (Exception ex) { GD.PrintErr($"Error in acquisition loop: {ex.Message}"); GD.PrintErr(ex.StackTrace); // Abort if captured frames are not appropriate if (ex is FormatException) { _recording = false; if (AcquisitionSource != null) { AcquisitionSource.CallDeferred("emit_signal", TestAcquisition.SignalName.RecordingError, ex.Message); } } return false; } } /// /// Update the screen space coordinates based on where the user is looking /// private void UpdateScreenSpace() { if (_rightEyeTransform == null) { GD.PrintErr("Right eye transform not initialized"); return; } Vector2 rightViewportPos = ProjectPointWithTransform(Target.GlobalTransform.Origin, _rightEyeTransform, _rightProjection); TargetInScreenSpace = rightViewportPos; } /// /// Main acquisition loop that runs in a separate thread. /// private void AcquisitionLoop() { try { while (_recording && !_cancellationTokenSource.Token.IsCancellationRequested) { // While recording is active, the screen space coordinates should always update for the enrollment scene UpdateScreenSpace(); // Skip processing if paused if (_paused) { Thread.Sleep(100); // Sleep briefly to avoid consuming CPU while paused continue; } if (!ProcessFrame(_cancellationTokenSource.Token)) { break; } } } catch (Exception ex) { GD.PrintErr($"Error in acquisition thread: {ex.Message}"); GD.PrintErr(ex.StackTrace); } finally { GD.Print("Acquisition thread exiting"); } } /// /// Starts the acquisition process. /// public async Task Start() { if (_recording) { GD.Print("Acquisition already running"); return false; } try { // Check required references if (VrCamera == null || XrOrigin == null || Target == null) { GD.PrintErr("Missing required references. Please assign VrCamera, XrOrigin, and Target in the Inspector."); throw new InvalidOperationException("Missing required references."); } // Setup camera bool cameraReady = await SetupCamera(); if (!cameraReady) { GD.PrintErr("Failed to initialize camera"); throw new InvalidOperationException("Unable to initialize eyetracking cameras.\nEnable camera access in Windows settings and close any app that is using the camera."); } // Initialize recording and visualization _dataRecorder.Start(); _visualizer.InitRerun(); // Reset counters _frameCount = 0; _lastEyeData = null; // Start acquisition thread _recording = true; _cancellationTokenSource = new CancellationTokenSource(); _acquisitionThread = new Thread(AcquisitionLoop); _acquisitionThread.Start(); GD.Print("Acquisition started"); return true; } catch (Exception ex) { GD.PrintErr($"Failed to start acquisition: {ex.Message}"); GD.PrintErr(ex.StackTrace); await Stop(); throw new InvalidOperationException(ex.Message); } } /// /// Stops the acquisition process. /// public async Task Stop() { _recording = false; // Signal the thread to stop and wait for it if (_cancellationTokenSource != null) { _cancellationTokenSource.Cancel(); } if (_acquisitionThread != null && _acquisitionThread.IsAlive) { // Wait for the thread to exit (with timeout) bool threadExited = _acquisitionThread.Join(2000); if (!threadExited) { GD.PrintErr("Acquisition thread did not exit cleanly"); // In a real application, you might need to handle this case better } _acquisitionThread = null; } // Clean up resources if (_cameraManager != null) { _cameraManager.ReleaseCamera(); } _dataRecorder.Cleanup(); GD.Print("Acquisition stopped"); // Wait a moment for resources to be released await Task.Delay(100); } /// /// Uploads the recorded data to the server. /// /// User ID for the server. /// Optional password for encryption. /// True if upload was successful, false otherwise. public async Task UploadData(string userId = "default", string password = null, UploadProgress progress = null) { try { // Make sure recording is stopped if (_recording) { await Stop(); } // Upload the data GD.Print($"Uploading data for user: {userId}"); string zipKey = await Task.Run(() => DataUploader.CompressAndUploadData(userId: userId, password: password, progress: progress)); bool success = !string.IsNullOrEmpty(zipKey); if (success) { GD.Print($"Data uploaded successfully to {zipKey}"); // Clear frames after upload _dataRecorder.ClearFrames(); // Enqueue a training job for the uploaded data GD.Print($"Enqueuing training job for data at: {zipKey}"); await TrainingJobManager.EnqueueTrainingJob(zipKey); } else { GD.PrintErr("Failed to upload data"); } return success; } catch (Exception ex) { GD.PrintErr($"Error uploading data: {ex.Message}"); GD.PrintErr(ex.StackTrace); throw new InvalidOperationException(ex.Message); } } private bool FetchHidDevice() { if (hidDevice == null) { // Find the HID device var deviceList = DeviceList.Local; hidDevice = deviceList.GetHidDevices(VID, PID).FirstOrDefault(); if (hidDevice == null) { GD.PrintErr($"HID device with VID:{VID:X4} PID:{PID:X4} not found"); return false; } GD.Print($"Found HID device: {hidDevice.GetFriendlyName()}"); } return true; } /// /// Sends hid brightness command to headset /// public bool ForceHeadsetBrightness(int brightness) { brightness = Math.Clamp(brightness, 6, 542); byte[] commandBytes = [ 0x00, 0x49, (byte)((brightness >> 8) & 0xFF), (byte)(brightness & 0xFF) ]; return SendFeatureCommand(commandBytes).Result; } /// /// Sends any feature command over hid /// public async Task SendFeatureCommand(byte[] commandOp) { try { if (!FetchHidDevice()) { return false; }; // Command bytes: [0, ord('e'), ord('R')] = [0, 101, 82] byte[] commandBytes = new byte[hidDevice.GetMaxFeatureReportLength()]; for (int i = 0; i < commandOp.Length; i++) { commandBytes[i] = commandOp[i]; } // Open the device and send the command using (var stream = hidDevice.Open()) { if (stream.CanWrite) { stream.SetFeature(commandBytes); return true; } else { GD.PrintErr("Cannot write to HID device"); return false; } } } catch (Exception ex) { GD.PrintErr($"Error sending feature command: {ex.Message}"); return false; } } private void CommandReadConfig(byte block) { byte[] commandBytes = [0x00, 0x55, block]; waiting_on_block = block; SendFeatureCommand(commandBytes); } /// /// Attempts to grab the current brightness value from flash memory /// public async Task GetCurrentBrightness() { if (!FetchHidDevice()) { return -1; }; requesting_config = true; int hid_read_timeout = 2000; // Read current config for (int b = 0; b < 16; ++b) { int hid_timeout_counter = 0; CommandReadConfig((byte)b); while (waiting_on_block != 0xFF) { await Task.Delay(1); if (hid_timeout_counter >= hid_read_timeout) return -1; hid_timeout_counter++; } } ParseHeadsetConfig(); requesting_config = false; for (int b = 0; b < headsetConfig.Count; b++) { if (headsetConfig[b].tag == 10) { byte[] bytes = System.Text.Encoding.ASCII.GetBytes(headsetConfig[b].data); short value = BitConverter.ToInt16(bytes, 0); return value; } } return -1; } private async Task HidLoop() { if (!FetchHidDevice()) { return; }; byte[] buffer = new byte[64]; long lastBRequest = 0; if (hidDevice != null && hidDevice.TryOpen(out HidStream stream)) { while (true) { int bytesRead = stream.Read(buffer, 0, buffer.Length); if (bytesRead > 0) { // Skip the report ID byte[] payload = new byte[64]; Array.Copy(buffer, 1, payload, 0, bytesRead - 1); if (payload[0] == 0x55 && requesting_config) { byte block = waiting_on_block; if (block * 32 + 32 > raw_config.Length) { GD.PrintErr("Response for block exceeded size: " + block.ToString()); break; } Buffer.BlockCopy(payload, 2, raw_config, block * 32, 32); waiting_on_block = 0xFF; } } // Force brightness periodically long current = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); if (current - lastBRequest > 4 && !AppQuitting) { lastBRequest = current; ForceHeadsetBrightness(244); // Translates to 90% from utility } await Task.Delay(1); } } } private void ParseHeadsetConfig() { headsetConfig.Clear(); for (int iter_b = 0; iter_b < CONFIG_DATA_LENGTH; iter_b++) { byte tag = raw_config[iter_b]; byte length = (iter_b < 511) ? raw_config[iter_b + 1] : (byte)0; if (tag == (byte)255) // End of valid tags { break; } string data = ""; for (int iter_c = 0; iter_c < length; iter_c++) { if ((iter_b + iter_c + 2) < CONFIG_DATA_LENGTH) { data += (char)raw_config[iter_b + iter_c + 2]; } } var entry = (tag, data); // Eliminate duplicates bool duplicate = false; for (int iter_t = 0; iter_t < headsetConfig.Count; iter_t++) { if (entry.tag == headsetConfig[iter_t].tag) { duplicate = true; break; } } if (!duplicate) { headsetConfig.Add(entry); } iter_b += 2 + length; // Skip tag + length + data } } /// /// Called when the node is about to be removed from the scene. /// public override void _ExitTree() { // Make sure to stop acquisition and clean up resources _ = Stop(); base._ExitTree(); } /// /// Pauses the acquisition process without stopping it completely. /// /// True if the acquisition was successfully paused, false otherwise. public bool Pause() { if (!_recording) { GD.Print("Cannot pause: Acquisition is not running"); return false; } if (_paused) { GD.Print("Acquisition is already paused"); return false; } _paused = true; GD.Print("Acquisition paused"); return true; } /// /// Resumes the acquisition process after it has been paused. /// /// True if the acquisition was successfully resumed, false otherwise. public bool Resume() { if (!_recording) { GD.Print("Cannot resume: Acquisition is not running"); return false; } if (!_paused) { GD.Print("Acquisition is not paused"); return false; } _paused = false; GD.Print("Acquisition resumed"); return true; } /// /// Gets whether the acquisition is currently paused. /// public bool IsPaused => _paused; }