using Godot;
using System;
using System.Threading;
using System.Threading.Tasks;
using OpenCvSharp;
using ETPreferences;
using HidSharp;
using System.Linq;
using System.Collections.Generic;
///
/// Processes eye tracking data.
///
public class EyeDataProcessor
{
///
/// Maximum age of eye data in seconds before it's considered stale.
///
public float MaxEyeDataAge { get; set; } = 0.1f;
///
/// Validates if eye data is fresh enough to use with the current frame.
///
/// The eye data to validate.
/// The timestamp of the current frame.
/// True if the eye data is valid for the current frame, false otherwise.
public bool ValidateEyeData(EyeData eyeData, float frameTimestamp)
{
if (eyeData == null)
return false;
// Check if the eye data is too old
float age = frameTimestamp - eyeData.Timestamp;
return age >= 0 && age <= MaxEyeDataAge;
}
}
///
/// Simple visualization for eye tracking data.
///
public class Visualizer
{
private bool _initialized = false;
///
/// Initializes the visualizer.
///
public void InitRerun()
{
_initialized = true;
GD.Print("Visualizer initialized");
}
///
/// Updates the visualization with the latest frame and eye data.
///
/// The current frame number.
/// The current frame as an OpenCV Mat.
/// The current eye position data.
public void UpdateVisualization(int frameCount, Mat frame, EyeData eyeData)
{
if (!_initialized)
{
GD.PrintErr("Visualizer not initialized. Call InitRerun() first.");
return;
}
// Draw eye positions on the frame for visualization
DrawEyePositions(frame, eyeData);
// In a real implementation, this would update a visualization window or stream
GD.Print($"Visualizing frame {frameCount} with eye positions: " +
$"L({eyeData.LeftPos.X:F2}, {eyeData.LeftPos.Y:F2}), " +
$"R({eyeData.RightPos.X:F2}, {eyeData.RightPos.Y:F2})");
}
///
/// Draws eye positions on the frame for visualization.
///
/// The frame to draw on.
/// The eye position data.
private void DrawEyePositions(Mat frame, EyeData eyeData)
{
// Convert normalized coordinates [0,1] to pixel coordinates
int leftX = (int)(eyeData.LeftPos.X * frame.Width);
int leftY = (int)(eyeData.LeftPos.Y * frame.Height);
int rightX = (int)(eyeData.RightPos.X * frame.Width);
int rightY = (int)(eyeData.RightPos.Y * frame.Height);
// Draw circles at eye positions using OpenCV
Cv2.Circle(frame, new Point(leftX, leftY), 10, new Scalar(0, 0, 255), -1); // Red circle for left eye
Cv2.Circle(frame, new Point(rightX, rightY), 10, new Scalar(255, 0, 0), -1); // Blue circle for right eye
}
}
///
/// Extended EyeData class with timestamp.
///
public partial class EyeData
{
///
/// Timestamp when the eye data was captured.
///
public float Timestamp { get; set; }
}
///
/// Manages the acquisition of eye tracking data and camera frames.
///
public partial class AcquisitionManager : Node3D
{
[Export]
public int FrameRate { get; set; } = 30;
[Export]
public Camera3D VrCamera { get; set; }
[Export]
public XROrigin3D XrOrigin { get; set; }
[Export]
public Node3D Target { get; set; }
[Export]
public TestAcquisition AcquisitionSource { get; set; }
[Export]
public float Distance { get; set; } = 1.5f;
public Vector2 TargetInScreenSpace { get; private set; }
public int DemoBlinkNotation { get; set; } = 0;
public ET_Preferences etSettings { get { return ET_Preferences.Instance; } }
public int OriginalHeadsetBrightness { get; set; } = -1;
public bool AppQuitting { get; set; }
private float _frameInterval;
private CameraManager _cameraManager;
private bool _recording = false;
private bool _paused = false;
private Thread _acquisitionThread;
private CancellationTokenSource _cancellationTokenSource;
private DataRecorder _dataRecorder;
private Visualizer _visualizer;
private EyeDataProcessor _eyeProcessor;
private int _frameCount = 0;
private EyeData _lastEyeData = null;
private Projection _leftProjection;
private Projection _rightProjection;
private Transform3D _leftEyeTransform;
private Transform3D _rightEyeTransform;
private bool _frustumRecorded = false;
private HidDevice hidDevice = null;
// Target device VID and PID
const int VID = 0x35bd;
const int PID = 0x0101;
private const uint CONFIG_DATA_LENGTH = 512;
private List<(byte tag, string data)> headsetConfig = new List<(byte tag, string data)>();
private byte[] raw_config = new byte[512];
private byte waiting_on_block;
private bool requesting_config;
///
/// Called when the node enters the scene tree for the first time.
///
public override void _Ready()
{
_frameInterval = 1.0f / FrameRate;
// Initialize components
_dataRecorder = new DataRecorder();
_visualizer = new Visualizer();
_eyeProcessor = new EyeDataProcessor();
// Create camera manager
_cameraManager = new CameraManager();
_cameraManager.DesiredWidth = 800;
_cameraManager.DesiredHeight = 400;
AddChild(_cameraManager);
// Fetch preferences
EyeTrackingPreferences.ReadSettingsFile();
// Ensure we have the required references
if (VrCamera == null)
{
GD.PrintErr("VR Camera reference is missing. Please assign it in the Inspector.");
}
if (XrOrigin == null)
{
GD.PrintErr("XR Origin reference is missing. Please assign it in the Inspector.");
}
if (Target == null)
{
GD.PrintErr("Target reference is missing. Please assign it in the Inspector.");
}
OpenXRManager.EnableOpenXR(this);
FetchHidDevice();
Task.Run(async () =>
{
HidLoop();
// Attempt to fetch the headset brightness to restore after enrollment
OriginalHeadsetBrightness = GetCurrentBrightness().Result;
GD.Print("Found headset brightness: " + OriginalHeadsetBrightness);
});
}
///
/// Called every frame to update the rig position.
///
public override void _Process(double delta)
{
if (_recording)
{
var xrInterface = XRServer.FindInterface("OpenXR");
if (xrInterface != null)
{
Vector2 viewport = xrInterface.GetRenderTargetSize();
float aspectRatio = viewport.X / viewport.Y;
_leftProjection = xrInterface.GetProjectionForView(0, aspectRatio, VrCamera.Near, VrCamera.Far);
_rightProjection = xrInterface.GetProjectionForView(1, aspectRatio, VrCamera.Near, VrCamera.Far);
// Update eye transforms in the main thread since they depend on current XR state
_leftEyeTransform = xrInterface.GetTransformForView(0, XrOrigin.GlobalTransform);
_rightEyeTransform = xrInterface.GetTransformForView(1, XrOrigin.GlobalTransform);
if (!_frustumRecorded)
{
_dataRecorder.RecordFrustum(_leftProjection, _rightProjection, VrCamera.Near, VrCamera.Far);
_frustumRecorded = true;
}
}
else
{
throw new Exception("OpenXR interface not found");
}
RepositionRig();
UpdateScreenSpace();
}
}
///
/// Repositions the rig based on the VR camera's forward direction.
///
private void RepositionRig()
{
if (Target == null || VrCamera == null)
{
GD.PrintErr("Target or VR Camera is not assigned.");
return;
}
// 1. Get the camera's current forward direction (world space)
Vector3 cameraForward = -VrCamera.GlobalTransform.Basis.Z;
// 2. We only want to use the horizontal component of the camera's forward direction
// This prevents vertical "headlocking" while maintaining the correct Z distance
Vector3 cameraForwardHorizontal = new Vector3(0, 0, cameraForward.Z).Normalized();
// 3. Calculate the desired position
Vector3 cameraLocalPos = VrCamera.Position; // local offset
Vector3 desiredCameraWorldPos = Target.GlobalPosition - cameraForwardHorizontal * Distance;
Vector3 newRigPosition = desiredCameraWorldPos - cameraLocalPos;
// 4. Move the XR Origin to that position
GlobalPosition = newRigPosition;
}
///
/// Sets up the camera for acquisition.
///
private async Task SetupCamera()
{
try
{
bool success = await _cameraManager.InitializeCamera();
if (!success)
{
GD.PrintErr("Failed to initialize camera");
return false;
}
GD.Print("Camera setup complete");
return true;
}
catch (Exception ex)
{
GD.PrintErr($"Error setting up camera: {ex.Message}");
GD.PrintErr(ex.StackTrace);
return false;
}
}
///
/// Gets eye tracking data directly from the VR system.
///
/// The latest eye data, or null if not available.
private EyeData GetEyeTrackingData()
{
try
{
if (VrCamera == null || XrOrigin == null || Target == null)
{
return null;
}
var xrInterface = XRServer.FindInterface("OpenXR");
if (xrInterface == null)
{
GD.PrintErr("OpenXR interface not found");
return null;
}
Vector2 viewport = xrInterface.GetRenderTargetSize();
float aspectRatio = viewport.X / viewport.Y;
// Use the stored transforms that were updated in _Process
Vector2 leftViewportPos = ProjectPointWithTransform(Target.GlobalTransform.Origin, _leftEyeTransform, _leftProjection);
Vector2 rightViewportPos = ProjectPointWithTransform(Target.GlobalTransform.Origin, _rightEyeTransform, _rightProjection);
// Create eye data with current timestamp
var eyeData = new EyeData(leftViewportPos, rightViewportPos);
eyeData.Timestamp = (float)Time.GetTicksMsec() / 1000.0f;
// Set eye openness
eyeData.LeftOpenness = DemoBlinkNotation;
eyeData.RightOpenness = DemoBlinkNotation;
return eyeData;
}
catch (Exception ex)
{
GD.PrintErr($"Error getting eye tracking data: {ex.Message}");
return null;
}
}
///
/// Projects a world point using a custom camera transform and projection matrix.
///
/// The point in world space to project.
/// The camera transform.
/// The projection matrix.
/// The projected point in viewport coordinates (range [0,1]).
private Vector2 ProjectPointWithTransform(Vector3 worldPoint, Transform3D cameraTransform, Projection projection)
{
// Get the view matrix (inverse of camera transform)
Transform3D viewMatrix = cameraTransform.Inverse();
// Transform the world point to view space
Vector3 pointInView = viewMatrix * worldPoint;
// Create a Vector4 for the projection
Vector4 vec = new Vector4(pointInView.X, pointInView.Y, pointInView.Z, 1.0f);
// Transform to clip space
Vector4 clipSpace = projection * vec;
// Convert to normalized device coordinates (NDC)
Vector3 ndc;
if (clipSpace.W != 0)
{
ndc = new Vector3(clipSpace.X, clipSpace.Y, clipSpace.Z) / clipSpace.W;
}
else
{
ndc = new Vector3(clipSpace.X, clipSpace.Y, clipSpace.Z);
}
// Convert from NDC (range [-1,1]) to viewport coordinates (range [0,1])
float viewportX = (ndc.X * 0.5f) + 0.5f;
float viewportY = (ndc.Y * 0.5f) + 0.5f;
return new Vector2(viewportX, viewportY);
}
///
/// Processes a single frame of data.
///
/// Token to monitor for cancellation requests.
/// True if processing should continue, false otherwise.
private bool ProcessFrame(CancellationToken cancellationToken)
{
float loopStart = (float)Time.GetTicksMsec() / 1000.0f;
try
{
// Check for cancellation
if (cancellationToken.IsCancellationRequested)
return false;
// Capture frame
Mat frame = _cameraManager.GetCameraFrame();
if (frame == null || frame.Empty())
{
GD.PrintErr("Failed to capture frame");
return true; // Continue trying
}
float frameTimestamp = (float)Time.GetTicksMsec() / 1000.0f;
_frameCount++;
// Get latest eye data
EyeData eyeData = GetEyeTrackingData();
if (eyeData != null)
{
_lastEyeData = eyeData;
GD.Print($"Updated eye data: L({_lastEyeData.LeftPos.X:F2}, {_lastEyeData.LeftPos.Y:F2}), " +
$"R({_lastEyeData.RightPos.X:F2}, {_lastEyeData.RightPos.Y:F2})");
}
// Process synchronized data
if (_lastEyeData != null && _eyeProcessor.ValidateEyeData(_lastEyeData, frameTimestamp))
{
GD.Print($"Synchronized frame {_frameCount} at {frameTimestamp:F3} " +
$"with eye data at {_lastEyeData.Timestamp:F3}");
// Record data
_dataRecorder.RecordFrame(frame, _frameCount);
_dataRecorder.RecordEyeData(_frameCount, frameTimestamp, _lastEyeData);
// Update visualization
_visualizer.UpdateVisualization(_frameCount, frame, _lastEyeData);
}
else
{
GD.Print($"No fresh eye data for frame {_frameCount} at {frameTimestamp:F3}");
}
// Maintain frame rate
float elapsed = ((float)Time.GetTicksMsec() / 1000.0f) - loopStart;
int sleepMs = (int)Math.Max(0, (_frameInterval - elapsed) * 1000);
if (sleepMs > 0)
{
Thread.Sleep(sleepMs);
}
return true;
}
catch (Exception ex)
{
GD.PrintErr($"Error in acquisition loop: {ex.Message}");
GD.PrintErr(ex.StackTrace);
// Abort if captured frames are not appropriate
if (ex is FormatException)
{
_recording = false;
if (AcquisitionSource != null)
{
AcquisitionSource.CallDeferred("emit_signal", TestAcquisition.SignalName.RecordingError, ex.Message);
}
}
return false;
}
}
///
/// Update the screen space coordinates based on where the user is looking
///
private void UpdateScreenSpace()
{
if (_rightEyeTransform == null)
{
GD.PrintErr("Right eye transform not initialized");
return;
}
Vector2 rightViewportPos = ProjectPointWithTransform(Target.GlobalTransform.Origin, _rightEyeTransform, _rightProjection);
TargetInScreenSpace = rightViewportPos;
}
///
/// Main acquisition loop that runs in a separate thread.
///
private void AcquisitionLoop()
{
try
{
while (_recording && !_cancellationTokenSource.Token.IsCancellationRequested)
{
// While recording is active, the screen space coordinates should always update for the enrollment scene
UpdateScreenSpace();
// Skip processing if paused
if (_paused)
{
Thread.Sleep(100); // Sleep briefly to avoid consuming CPU while paused
continue;
}
if (!ProcessFrame(_cancellationTokenSource.Token))
{
break;
}
}
}
catch (Exception ex)
{
GD.PrintErr($"Error in acquisition thread: {ex.Message}");
GD.PrintErr(ex.StackTrace);
}
finally
{
GD.Print("Acquisition thread exiting");
}
}
///
/// Starts the acquisition process.
///
public async Task Start()
{
if (_recording)
{
GD.Print("Acquisition already running");
return false;
}
try
{
// Check required references
if (VrCamera == null || XrOrigin == null || Target == null)
{
GD.PrintErr("Missing required references. Please assign VrCamera, XrOrigin, and Target in the Inspector.");
throw new InvalidOperationException("Missing required references.");
}
// Setup camera
bool cameraReady = await SetupCamera();
if (!cameraReady)
{
GD.PrintErr("Failed to initialize camera");
throw new InvalidOperationException("Unable to initialize eyetracking cameras.\nEnable camera access in Windows settings and close any app that is using the camera.");
}
// Initialize recording and visualization
_dataRecorder.Start();
_visualizer.InitRerun();
// Reset counters
_frameCount = 0;
_lastEyeData = null;
// Start acquisition thread
_recording = true;
_cancellationTokenSource = new CancellationTokenSource();
_acquisitionThread = new Thread(AcquisitionLoop);
_acquisitionThread.Start();
GD.Print("Acquisition started");
return true;
}
catch (Exception ex)
{
GD.PrintErr($"Failed to start acquisition: {ex.Message}");
GD.PrintErr(ex.StackTrace);
await Stop();
throw new InvalidOperationException(ex.Message);
}
}
///
/// Stops the acquisition process.
///
public async Task Stop()
{
_recording = false;
// Signal the thread to stop and wait for it
if (_cancellationTokenSource != null)
{
_cancellationTokenSource.Cancel();
}
if (_acquisitionThread != null && _acquisitionThread.IsAlive)
{
// Wait for the thread to exit (with timeout)
bool threadExited = _acquisitionThread.Join(2000);
if (!threadExited)
{
GD.PrintErr("Acquisition thread did not exit cleanly");
// In a real application, you might need to handle this case better
}
_acquisitionThread = null;
}
// Clean up resources
if (_cameraManager != null)
{
_cameraManager.ReleaseCamera();
}
_dataRecorder.Cleanup();
GD.Print("Acquisition stopped");
// Wait a moment for resources to be released
await Task.Delay(100);
}
///
/// Uploads the recorded data to the server.
///
/// User ID for the server.
/// Optional password for encryption.
/// True if upload was successful, false otherwise.
public async Task UploadData(string userId = "default", string password = null, UploadProgress progress = null)
{
try
{
// Make sure recording is stopped
if (_recording)
{
await Stop();
}
// Upload the data
GD.Print($"Uploading data for user: {userId}");
string zipKey = await Task.Run(() => DataUploader.CompressAndUploadData(userId: userId, password: password, progress: progress));
bool success = !string.IsNullOrEmpty(zipKey);
if (success)
{
GD.Print($"Data uploaded successfully to {zipKey}");
// Clear frames after upload
_dataRecorder.ClearFrames();
// Enqueue a training job for the uploaded data
GD.Print($"Enqueuing training job for data at: {zipKey}");
await TrainingJobManager.EnqueueTrainingJob(zipKey);
}
else
{
GD.PrintErr("Failed to upload data");
}
return success;
}
catch (Exception ex)
{
GD.PrintErr($"Error uploading data: {ex.Message}");
GD.PrintErr(ex.StackTrace);
throw new InvalidOperationException(ex.Message);
}
}
private bool FetchHidDevice()
{
if (hidDevice == null)
{
// Find the HID device
var deviceList = DeviceList.Local;
hidDevice = deviceList.GetHidDevices(VID, PID).FirstOrDefault();
if (hidDevice == null)
{
GD.PrintErr($"HID device with VID:{VID:X4} PID:{PID:X4} not found");
return false;
}
GD.Print($"Found HID device: {hidDevice.GetFriendlyName()}");
}
return true;
}
///
/// Sends hid brightness command to headset
///
public bool ForceHeadsetBrightness(int brightness)
{
brightness = Math.Clamp(brightness, 6, 542);
byte[] commandBytes = [
0x00,
0x49,
(byte)((brightness >> 8) & 0xFF),
(byte)(brightness & 0xFF)
];
return SendFeatureCommand(commandBytes).Result;
}
///
/// Sends any feature command over hid
///
public async Task SendFeatureCommand(byte[] commandOp)
{
try
{
if (!FetchHidDevice())
{
return false;
};
// Command bytes: [0, ord('e'), ord('R')] = [0, 101, 82]
byte[] commandBytes = new byte[hidDevice.GetMaxFeatureReportLength()];
for (int i = 0; i < commandOp.Length; i++)
{
commandBytes[i] = commandOp[i];
}
// Open the device and send the command
using (var stream = hidDevice.Open())
{
if (stream.CanWrite)
{
stream.SetFeature(commandBytes);
return true;
}
else
{
GD.PrintErr("Cannot write to HID device");
return false;
}
}
}
catch (Exception ex)
{
GD.PrintErr($"Error sending feature command: {ex.Message}");
return false;
}
}
private void CommandReadConfig(byte block)
{
byte[] commandBytes = [0x00, 0x55, block];
waiting_on_block = block;
SendFeatureCommand(commandBytes);
}
///
/// Attempts to grab the current brightness value from flash memory
///
public async Task GetCurrentBrightness()
{
if (!FetchHidDevice())
{
return -1;
};
requesting_config = true;
int hid_read_timeout = 2000;
// Read current config
for (int b = 0; b < 16; ++b)
{
int hid_timeout_counter = 0;
CommandReadConfig((byte)b);
while (waiting_on_block != 0xFF)
{
await Task.Delay(1);
if (hid_timeout_counter >= hid_read_timeout)
return -1;
hid_timeout_counter++;
}
}
ParseHeadsetConfig();
requesting_config = false;
for (int b = 0; b < headsetConfig.Count; b++)
{
if (headsetConfig[b].tag == 10)
{
byte[] bytes = System.Text.Encoding.ASCII.GetBytes(headsetConfig[b].data);
short value = BitConverter.ToInt16(bytes, 0);
return value;
}
}
return -1;
}
private async Task HidLoop()
{
if (!FetchHidDevice())
{
return;
};
byte[] buffer = new byte[64];
long lastBRequest = 0;
if (hidDevice != null && hidDevice.TryOpen(out HidStream stream))
{
while (true)
{
int bytesRead = stream.Read(buffer, 0, buffer.Length);
if (bytesRead > 0)
{
// Skip the report ID
byte[] payload = new byte[64];
Array.Copy(buffer, 1, payload, 0, bytesRead - 1);
if (payload[0] == 0x55 && requesting_config)
{
byte block = waiting_on_block;
if (block * 32 + 32 > raw_config.Length)
{
GD.PrintErr("Response for block exceeded size: " + block.ToString());
break;
}
Buffer.BlockCopy(payload, 2, raw_config, block * 32, 32);
waiting_on_block = 0xFF;
}
}
// Force brightness periodically
long current = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
if (current - lastBRequest > 4 && !AppQuitting)
{
lastBRequest = current;
ForceHeadsetBrightness(244); // Translates to 90% from utility
}
await Task.Delay(1);
}
}
}
private void ParseHeadsetConfig()
{
headsetConfig.Clear();
for (int iter_b = 0; iter_b < CONFIG_DATA_LENGTH; iter_b++)
{
byte tag = raw_config[iter_b];
byte length = (iter_b < 511) ? raw_config[iter_b + 1] : (byte)0;
if (tag == (byte)255) // End of valid tags
{
break;
}
string data = "";
for (int iter_c = 0; iter_c < length; iter_c++)
{
if ((iter_b + iter_c + 2) < CONFIG_DATA_LENGTH)
{
data += (char)raw_config[iter_b + iter_c + 2];
}
}
var entry = (tag, data);
// Eliminate duplicates
bool duplicate = false;
for (int iter_t = 0; iter_t < headsetConfig.Count; iter_t++)
{
if (entry.tag == headsetConfig[iter_t].tag)
{
duplicate = true;
break;
}
}
if (!duplicate)
{
headsetConfig.Add(entry);
}
iter_b += 2 + length; // Skip tag + length + data
}
}
///
/// Called when the node is about to be removed from the scene.
///
public override void _ExitTree()
{
// Make sure to stop acquisition and clean up resources
_ = Stop();
base._ExitTree();
}
///
/// Pauses the acquisition process without stopping it completely.
///
/// True if the acquisition was successfully paused, false otherwise.
public bool Pause()
{
if (!_recording)
{
GD.Print("Cannot pause: Acquisition is not running");
return false;
}
if (_paused)
{
GD.Print("Acquisition is already paused");
return false;
}
_paused = true;
GD.Print("Acquisition paused");
return true;
}
///
/// Resumes the acquisition process after it has been paused.
///
/// True if the acquisition was successfully resumed, false otherwise.
public bool Resume()
{
if (!_recording)
{
GD.Print("Cannot resume: Acquisition is not running");
return false;
}
if (!_paused)
{
GD.Print("Acquisition is not paused");
return false;
}
_paused = false;
GD.Print("Acquisition resumed");
return true;
}
///
/// Gets whether the acquisition is currently paused.
///
public bool IsPaused => _paused;
}