using Godot; using System; using ETPreferences; /// /// Test scene for visualizing I-VT filtering. /// Shows two cursors: one for raw gaze data and one for filtered gaze data. /// public partial class IVTTestScene : Node3D { [Export] public NodePath RawCursorPath { get; set; } [Export] public NodePath FilteredCursorPath { get; set; } [Export] public NodePath InfoLabelPath { get; set; } [Export] public float CursorDistance { get; set; } = 2.0f; // Distance from camera in meters [Export] public float CursorSize { get; set; } = 0.08f; // Size of cursor spheres (radius) private MeshInstance3D _rawCursor; private MeshInstance3D _filteredCursor; private Label3D _infoLabel; private XRCamera3D _xrCamera; private GazeEstimation _gazeEstimation; private IVTTestConsumer _testConsumer; private Vector3 _lastRawCombined = Vector3.Zero; private Vector3 _lastFiltered = Vector3.Zero; private bool _lastInSaccade = false; private int _updateCount = 0; private float _totalAngularDiff = 0f; public override void _Ready() { // Enable OpenXR OpenXRManager.EnableOpenXR(this); // Find XR camera _xrCamera = GetNode("XROrigin3D/XRCamera3D"); // Create cursors if not assigned if (RawCursorPath == null || RawCursorPath.IsEmpty) { _rawCursor = CreateCursor(new Color(1, 0, 0, 0.8f), "RawCursor"); // Red } else { _rawCursor = GetNode(RawCursorPath); } if (FilteredCursorPath == null || FilteredCursorPath.IsEmpty) { _filteredCursor = CreateCursor(new Color(0, 1, 0, 0.8f), "FilteredCursor"); // Green } else { _filteredCursor = GetNode(FilteredCursorPath); } // Create info label if (InfoLabelPath == null || InfoLabelPath.IsEmpty) { _infoLabel = CreateInfoLabel(); } else { _infoLabel = GetNode(InfoLabelPath); } // Initialize eye tracking EyeTrackingPreferences.ReadSettingsFile(); InitializeEyeTracking(); GD.Print("IVT Test Scene ready. Red cursor = Raw gaze, Green cursor = Filtered gaze"); } private MeshInstance3D CreateCursor(Color color, string name) { var cursor = new MeshInstance3D(); cursor.Name = name; // Use BoxMesh instead of SphereMesh to avoid mesh generation issues var boxMesh = new BoxMesh(); boxMesh.Size = new Vector3(CursorSize * 2, CursorSize * 2, CursorSize * 2); // Create material with emission var material = new StandardMaterial3D(); material.AlbedoColor = color; material.ShadingMode = BaseMaterial3D.ShadingModeEnum.Unshaded; material.Transparency = BaseMaterial3D.TransparencyEnum.Alpha; // Add emission to make cursors glow and more visible material.EmissionEnabled = true; material.Emission = new Color(color.R, color.G, color.B, 1.0f); material.EmissionEnergyMultiplier = 3.0f; // Increased for better visibility // Assign mesh to instance cursor.Mesh = boxMesh; // Apply material override cursor.MaterialOverride = material; // Set initial position (straight ahead) so cursors are visible before gaze data arrives cursor.Position = new Vector3(0, 0, -CursorDistance); AddChild(cursor); GD.Print($"Created cursor '{name}' (Box) at position {cursor.Position} with size {CursorSize * 2}"); return cursor; } private Label3D CreateInfoLabel() { var label = new Label3D(); label.Name = "InfoLabel"; label.Text = "Initializing..."; label.FontSize = 32; label.Modulate = new Color(1, 1, 1, 1); //label.Billboard = BaseMaterial3D.BillboardModeEnum.Enabled; // Position below the cursors label.Position = new Vector3(0, 1.5f, -5.0f); AddChild(label); return label; } private async void InitializeEyeTracking() { try { // Create gaze estimation system _gazeEstimation = new GazeEstimation(); AddChild(_gazeEstimation); // Initialize camera bool success = await _gazeEstimation.cameraManager.InitializeCamera(); if (success) { // Load model if available var etSettings = ET_Preferences.Instance; if (etSettings != null && !string.IsNullOrEmpty(etSettings.selected_model)) { string modelPath = etSettings.selected_model; // Check if model path is an S3 URL if (modelPath.StartsWith("s3://")) { // Try to find a local downloaded version bool foundLocal = false; if (etSettings.downloaded_models != null && etSettings.downloaded_models.Count > 0) { // Use the first downloaded model as fallback modelPath = etSettings.downloaded_models[0]; foundLocal = true; GD.Print($"S3 model detected, using local model: {modelPath}"); } if (!foundLocal) { GD.PrintErr("Selected model is S3 URL with no local download. Please download a model first."); UpdateInfoLabel("Model not downloaded!\nPlease download in main app", new Color(1, 0, 0)); modelPath = null; } } if (!string.IsNullOrEmpty(modelPath)) { GD.Print($"Loading model: {modelPath}"); _gazeEstimation.LoadModel(modelPath); UpdateInfoLabel("Loading model...\nRed: Raw | Green: Filtered", new Color(1, 1, 0)); } } else { GD.PrintErr("No model selected! Please select a model in the main application first."); UpdateInfoLabel("No model selected!\nPlease select model in main app", new Color(1, 0, 0)); } // Create test consumer _testConsumer = new IVTTestConsumer(); _testConsumer.XrCamera = _xrCamera; // Assign XR camera for head tracking _testConsumer.RawGazeUpdated += OnRawGazeUpdated; _testConsumer.FilteredGazeUpdated += OnFilteredGazeUpdated; AddChild(_testConsumer); // Explicitly enable the consumer _testConsumer.Enable(); // Register consumer with gaze estimation _gazeEstimation.RegisterObserver(_testConsumer); GD.Print($"IVT Test Consumer initialized and registered (Enabled: {_testConsumer.Enabled})"); GD.Print($"GazeEstimation IsRunning: {_gazeEstimation.IsRunning}"); UpdateInfoLabel("Eye tracking initialized\nRed: Raw | Green: Filtered", new Color(0, 1, 0)); } else { GD.PrintErr("Failed to initialize camera"); UpdateInfoLabel("Failed to initialize camera\nCheck camera permissions", new Color(1, 0, 0)); // Start camera search _gazeEstimation.cameraManager.StartCameraSearch(); } } catch (Exception ex) { GD.PrintErr($"Error initializing eye tracking: {ex.Message}"); GD.PrintErr(ex.StackTrace); UpdateInfoLabel($"Error: {ex.Message}", new Color(1, 0, 0)); } } private void OnRawGazeUpdated(Vector3 leftVector, Vector3 rightVector, Vector3 combinedVector) { _lastRawCombined = combinedVector; } private void OnFilteredGazeUpdated(Vector3 combinedVector, bool inSaccade) { _lastFiltered = combinedVector; _lastInSaccade = inSaccade; // Calculate angular difference if (_lastRawCombined != Vector3.Zero && _lastFiltered != Vector3.Zero) { float dotProduct = _lastRawCombined.Normalized().Dot(_lastFiltered.Normalized()); dotProduct = Mathf.Clamp(dotProduct, -1.0f, 1.0f); float angleDegrees = Mathf.RadToDeg(Mathf.Acos(dotProduct)); _totalAngularDiff += angleDegrees; _updateCount++; } } public override void _Input(InputEvent @event) { base._Input(@event); // Handle keypress for recording if (@event is InputEventKey keyEvent && keyEvent.Pressed) { // Press 'R' to start/stop recording if (keyEvent.Keycode == Key.R && _testConsumer != null) { if (_testConsumer.IsRecording) { _testConsumer.StopRecording(); UpdateInfoLabel($"Recording stopped. Saved {_testConsumer.RecordedFrameCount} frames.\nPress R to start recording", new Color(1, 1, 0)); } else { _testConsumer.StartRecording(); UpdateInfoLabel($"Recording started... ({_testConsumer.RecordedFrameCount}/{IVTTestConsumer.MAX_RECORDING_FRAMES} frames)\nPress R to stop", new Color(0, 1, 0)); } } } } public override void _Process(double delta) { base._Process(delta); // Update recording status in info label if recording if (_testConsumer != null && _testConsumer.IsRecording) { UpdateInfoLabel($"Recording... ({_testConsumer.RecordedFrameCount}/{IVTTestConsumer.MAX_RECORDING_FRAMES} frames)\nPress R to stop", new Color(0, 1, 0)); } // Project cursors along 3D gaze direction, CursorDistance meters from the camera if (_xrCamera != null) { Vector3 cameraGlobalPos = _xrCamera.GlobalPosition; Basis cameraBasis = _xrCamera.GlobalTransform.Basis; // Raw cursor from raw combined gaze vector if (_rawCursor != null && _lastRawCombined != Vector3.Zero) { Vector3 localDir = _lastRawCombined.Normalized(); Vector3 worldDir = cameraBasis * localDir; _rawCursor.GlobalPosition = cameraGlobalPos + worldDir * CursorDistance; } // Filtered cursor from filtered combined gaze vector if (_filteredCursor != null && _lastFiltered != Vector3.Zero) { Vector3 localDir = _lastFiltered.Normalized(); Vector3 worldDir = cameraBasis * localDir; _filteredCursor.GlobalPosition = cameraGlobalPos + worldDir * CursorDistance; } } // Update info label text if (_infoLabel != null && _updateCount > 0) { float avgDiff = _totalAngularDiff / _updateCount; string saccadeStatus = _lastInSaccade ? "SACCADE" : "FIXATION"; Color statusColor = _lastInSaccade ? new Color(1, 0.5f, 0) : new Color(0, 1, 0); string info = $"Red: Raw Gaze | Green: Filtered\n" + $"Status: {saccadeStatus}\n" + $"Avg Difference: {avgDiff:F2}°\n" + $"Press ESC to exit"; UpdateInfoLabel(info, statusColor); } } private void UpdateInfoLabel(string text, Color? color = null) { if (_infoLabel != null) { _infoLabel.Text = text; if (color.HasValue) { _infoLabel.Modulate = color.Value; } } } public override void _ExitTree() { base._ExitTree(); // Clean up if (_testConsumer != null) { _testConsumer.RawGazeUpdated -= OnRawGazeUpdated; _testConsumer.FilteredGazeUpdated -= OnFilteredGazeUpdated; if (_gazeEstimation != null) { _gazeEstimation.RemoveObserver(_testConsumer); } } GD.Print("IVT Test Scene cleaned up"); } }