using Godot;
using System;
using ETPreferences;
///
/// Test scene for visualizing I-VT filtering.
/// Shows two cursors: one for raw gaze data and one for filtered gaze data.
///
public partial class IVTTestScene : Node3D
{
[Export]
public NodePath RawCursorPath { get; set; }
[Export]
public NodePath FilteredCursorPath { get; set; }
[Export]
public NodePath InfoLabelPath { get; set; }
[Export]
public float CursorDistance { get; set; } = 2.0f; // Distance from camera in meters
[Export]
public float CursorSize { get; set; } = 0.08f; // Size of cursor spheres (radius)
private MeshInstance3D _rawCursor;
private MeshInstance3D _filteredCursor;
private Label3D _infoLabel;
private XRCamera3D _xrCamera;
private GazeEstimation _gazeEstimation;
private IVTTestConsumer _testConsumer;
private Vector3 _lastRawCombined = Vector3.Zero;
private Vector3 _lastFiltered = Vector3.Zero;
private bool _lastInSaccade = false;
private int _updateCount = 0;
private float _totalAngularDiff = 0f;
public override void _Ready()
{
// Enable OpenXR
OpenXRManager.EnableOpenXR(this);
// Find XR camera
_xrCamera = GetNode("XROrigin3D/XRCamera3D");
// Create cursors if not assigned
if (RawCursorPath == null || RawCursorPath.IsEmpty)
{
_rawCursor = CreateCursor(new Color(1, 0, 0, 0.8f), "RawCursor"); // Red
}
else
{
_rawCursor = GetNode(RawCursorPath);
}
if (FilteredCursorPath == null || FilteredCursorPath.IsEmpty)
{
_filteredCursor = CreateCursor(new Color(0, 1, 0, 0.8f), "FilteredCursor"); // Green
}
else
{
_filteredCursor = GetNode(FilteredCursorPath);
}
// Create info label
if (InfoLabelPath == null || InfoLabelPath.IsEmpty)
{
_infoLabel = CreateInfoLabel();
}
else
{
_infoLabel = GetNode(InfoLabelPath);
}
// Initialize eye tracking
EyeTrackingPreferences.ReadSettingsFile();
InitializeEyeTracking();
GD.Print("IVT Test Scene ready. Red cursor = Raw gaze, Green cursor = Filtered gaze");
}
private MeshInstance3D CreateCursor(Color color, string name)
{
var cursor = new MeshInstance3D();
cursor.Name = name;
// Use BoxMesh instead of SphereMesh to avoid mesh generation issues
var boxMesh = new BoxMesh();
boxMesh.Size = new Vector3(CursorSize * 2, CursorSize * 2, CursorSize * 2);
// Create material with emission
var material = new StandardMaterial3D();
material.AlbedoColor = color;
material.ShadingMode = BaseMaterial3D.ShadingModeEnum.Unshaded;
material.Transparency = BaseMaterial3D.TransparencyEnum.Alpha;
// Add emission to make cursors glow and more visible
material.EmissionEnabled = true;
material.Emission = new Color(color.R, color.G, color.B, 1.0f);
material.EmissionEnergyMultiplier = 3.0f; // Increased for better visibility
// Assign mesh to instance
cursor.Mesh = boxMesh;
// Apply material override
cursor.MaterialOverride = material;
// Set initial position (straight ahead) so cursors are visible before gaze data arrives
cursor.Position = new Vector3(0, 0, -CursorDistance);
AddChild(cursor);
GD.Print($"Created cursor '{name}' (Box) at position {cursor.Position} with size {CursorSize * 2}");
return cursor;
}
private Label3D CreateInfoLabel()
{
var label = new Label3D();
label.Name = "InfoLabel";
label.Text = "Initializing...";
label.FontSize = 32;
label.Modulate = new Color(1, 1, 1, 1);
//label.Billboard = BaseMaterial3D.BillboardModeEnum.Enabled;
// Position below the cursors
label.Position = new Vector3(0, 1.5f, -5.0f);
AddChild(label);
return label;
}
private async void InitializeEyeTracking()
{
try
{
// Create gaze estimation system
_gazeEstimation = new GazeEstimation();
AddChild(_gazeEstimation);
// Initialize camera
bool success = await _gazeEstimation.cameraManager.InitializeCamera();
if (success)
{
// Load model if available
var etSettings = ET_Preferences.Instance;
if (etSettings != null && !string.IsNullOrEmpty(etSettings.selected_model))
{
string modelPath = etSettings.selected_model;
// Check if model path is an S3 URL
if (modelPath.StartsWith("s3://"))
{
// Try to find a local downloaded version
bool foundLocal = false;
if (etSettings.downloaded_models != null && etSettings.downloaded_models.Count > 0)
{
// Use the first downloaded model as fallback
modelPath = etSettings.downloaded_models[0];
foundLocal = true;
GD.Print($"S3 model detected, using local model: {modelPath}");
}
if (!foundLocal)
{
GD.PrintErr("Selected model is S3 URL with no local download. Please download a model first.");
UpdateInfoLabel("Model not downloaded!\nPlease download in main app", new Color(1, 0, 0));
modelPath = null;
}
}
if (!string.IsNullOrEmpty(modelPath))
{
GD.Print($"Loading model: {modelPath}");
_gazeEstimation.LoadModel(modelPath);
UpdateInfoLabel("Loading model...\nRed: Raw | Green: Filtered", new Color(1, 1, 0));
}
}
else
{
GD.PrintErr("No model selected! Please select a model in the main application first.");
UpdateInfoLabel("No model selected!\nPlease select model in main app", new Color(1, 0, 0));
}
// Create test consumer
_testConsumer = new IVTTestConsumer();
_testConsumer.XrCamera = _xrCamera; // Assign XR camera for head tracking
_testConsumer.RawGazeUpdated += OnRawGazeUpdated;
_testConsumer.FilteredGazeUpdated += OnFilteredGazeUpdated;
AddChild(_testConsumer);
// Explicitly enable the consumer
_testConsumer.Enable();
// Register consumer with gaze estimation
_gazeEstimation.RegisterObserver(_testConsumer);
GD.Print($"IVT Test Consumer initialized and registered (Enabled: {_testConsumer.Enabled})");
GD.Print($"GazeEstimation IsRunning: {_gazeEstimation.IsRunning}");
UpdateInfoLabel("Eye tracking initialized\nRed: Raw | Green: Filtered", new Color(0, 1, 0));
}
else
{
GD.PrintErr("Failed to initialize camera");
UpdateInfoLabel("Failed to initialize camera\nCheck camera permissions", new Color(1, 0, 0));
// Start camera search
_gazeEstimation.cameraManager.StartCameraSearch();
}
}
catch (Exception ex)
{
GD.PrintErr($"Error initializing eye tracking: {ex.Message}");
GD.PrintErr(ex.StackTrace);
UpdateInfoLabel($"Error: {ex.Message}", new Color(1, 0, 0));
}
}
private void OnRawGazeUpdated(Vector3 leftVector, Vector3 rightVector, Vector3 combinedVector)
{
_lastRawCombined = combinedVector;
}
private void OnFilteredGazeUpdated(Vector3 combinedVector, bool inSaccade)
{
_lastFiltered = combinedVector;
_lastInSaccade = inSaccade;
// Calculate angular difference
if (_lastRawCombined != Vector3.Zero && _lastFiltered != Vector3.Zero)
{
float dotProduct = _lastRawCombined.Normalized().Dot(_lastFiltered.Normalized());
dotProduct = Mathf.Clamp(dotProduct, -1.0f, 1.0f);
float angleDegrees = Mathf.RadToDeg(Mathf.Acos(dotProduct));
_totalAngularDiff += angleDegrees;
_updateCount++;
}
}
public override void _Input(InputEvent @event)
{
base._Input(@event);
// Handle keypress for recording
if (@event is InputEventKey keyEvent && keyEvent.Pressed)
{
// Press 'R' to start/stop recording
if (keyEvent.Keycode == Key.R && _testConsumer != null)
{
if (_testConsumer.IsRecording)
{
_testConsumer.StopRecording();
UpdateInfoLabel($"Recording stopped. Saved {_testConsumer.RecordedFrameCount} frames.\nPress R to start recording", new Color(1, 1, 0));
}
else
{
_testConsumer.StartRecording();
UpdateInfoLabel($"Recording started... ({_testConsumer.RecordedFrameCount}/{IVTTestConsumer.MAX_RECORDING_FRAMES} frames)\nPress R to stop", new Color(0, 1, 0));
}
}
}
}
public override void _Process(double delta)
{
base._Process(delta);
// Update recording status in info label if recording
if (_testConsumer != null && _testConsumer.IsRecording)
{
UpdateInfoLabel($"Recording... ({_testConsumer.RecordedFrameCount}/{IVTTestConsumer.MAX_RECORDING_FRAMES} frames)\nPress R to stop", new Color(0, 1, 0));
}
// Project cursors along 3D gaze direction, CursorDistance meters from the camera
if (_xrCamera != null)
{
Vector3 cameraGlobalPos = _xrCamera.GlobalPosition;
Basis cameraBasis = _xrCamera.GlobalTransform.Basis;
// Raw cursor from raw combined gaze vector
if (_rawCursor != null && _lastRawCombined != Vector3.Zero)
{
Vector3 localDir = _lastRawCombined.Normalized();
Vector3 worldDir = cameraBasis * localDir;
_rawCursor.GlobalPosition = cameraGlobalPos + worldDir * CursorDistance;
}
// Filtered cursor from filtered combined gaze vector
if (_filteredCursor != null && _lastFiltered != Vector3.Zero)
{
Vector3 localDir = _lastFiltered.Normalized();
Vector3 worldDir = cameraBasis * localDir;
_filteredCursor.GlobalPosition = cameraGlobalPos + worldDir * CursorDistance;
}
}
// Update info label text
if (_infoLabel != null && _updateCount > 0)
{
float avgDiff = _totalAngularDiff / _updateCount;
string saccadeStatus = _lastInSaccade ? "SACCADE" : "FIXATION";
Color statusColor = _lastInSaccade ? new Color(1, 0.5f, 0) : new Color(0, 1, 0);
string info = $"Red: Raw Gaze | Green: Filtered\n" +
$"Status: {saccadeStatus}\n" +
$"Avg Difference: {avgDiff:F2}°\n" +
$"Press ESC to exit";
UpdateInfoLabel(info, statusColor);
}
}
private void UpdateInfoLabel(string text, Color? color = null)
{
if (_infoLabel != null)
{
_infoLabel.Text = text;
if (color.HasValue)
{
_infoLabel.Modulate = color.Value;
}
}
}
public override void _ExitTree()
{
base._ExitTree();
// Clean up
if (_testConsumer != null)
{
_testConsumer.RawGazeUpdated -= OnRawGazeUpdated;
_testConsumer.FilteredGazeUpdated -= OnFilteredGazeUpdated;
if (_gazeEstimation != null)
{
_gazeEstimation.RemoveObserver(_testConsumer);
}
}
GD.Print("IVT Test Scene cleaned up");
}
}