using Godot; using System; using ETPreferences; using System.Collections.Generic; using System.Threading; /// /// Handling alignment scene events and its consumer /// public partial class AlignmentScene : Node3D { [Export] public MeshInstance3D InfoText { get; set; } [Export] public MeshInstance3D SubText { get; set; } [Export] public MeshInstance3D OuterRing { get; set; } [Export] public MeshInstance3D InnerRing { get; set; } [Export] public float CursorDistance { get; set; } = 2.0f; // Distance from camera in meters [Export] public float CursorSize { get; set; } = 0.05f; // Size of cursor spheres (radius) [Export] public float InnerRingRange { get; set; } [Export] public float OuterRingRange { get; set; } [Export] Color InnerColorActive = new Color(0.2f, 0.7f, 0.2f); [Export] Color InnerColorInactive = new Color(0.5f, 0.5f, 0.5f); [Export] Color OuterColorActive = new Color(0.7f, 0.7f, 0.2f); [Export] Color OuterColorInactive = new Color(0.5f, 0.5f, 0.5f); [Export] public ProgCircle ProgCircle { get; set; } private struct RingState { public bool active; public bool previouslyActive; public float targetThreshold; public float hitPitch; public AudioStreamPlayer hitSfx; public MeshInstance3D ringInstance; public Color activeColor; public Color inactiveColor; } private MeshInstance3D _rawCursor; private XRCamera3D _xrCamera; private GazeEstimation _gazeEstimation; private AlignmentConsumer _alignConsumer; private TextMesh _infoText; private TextMesh _subText; private Vector3 _lastRawCombined = Vector3.Zero; private int _updateCount = 0; private float _totalAngularDiff = 0f; private const string alignInstruction = "Focus on the dot in the center of the target.\nAdjust your headset on your face while focusing on the dot\nso that the red cursor remains inside the inner ring."; private const string closeInstruction = "Hold your position.."; private List target_rings; // Closing states bool startedCloseProgress = false; bool closingInProgress = false; bool allRingsActive = true; bool quitPending = false; float closeProgress = 0; float closeDuration = 2; // 2 seconds float closeBeginDelay = 4; // 4 seconds private CancellationTokenSource closingCancellation; public override void _Ready() { // Define target rings target_rings = new List() { new RingState() { hitPitch = 1f, ringInstance = InnerRing, targetThreshold = InnerRingRange, activeColor = InnerColorActive, inactiveColor = InnerColorInactive }, new RingState() { hitPitch = 0.83f, ringInstance = OuterRing, targetThreshold = OuterRingRange, activeColor = OuterColorActive, inactiveColor = OuterColorInactive } }; UpdateTarget(new Vector3(1, 1, 1)); // Create ring sfx for (int i = 0; i < target_rings.Count; i++) { RingState ring = target_rings[i]; var newSfx = new AudioStreamPlayer(); newSfx.PitchScale = ring.hitPitch; AddChild(newSfx); var audioStream = GD.Load("res://sfx/ALIGN-SFX-HIT.wav"); if (audioStream != null) { newSfx.Stream = audioStream; ring.hitSfx = newSfx; } target_rings[i] = ring; } ProgCircle.Progress = 0; // Enable OpenXR OpenXRManager.EnableOpenXR(this); // Find XR camera _xrCamera = GetNode("XROrigin3D/XRCamera3D"); // Create cursors if not assigned _rawCursor = CreateCursor(new Color(1, 0, 0, 0.8f), "RawCursor"); // Red // Scene's text labels _infoText = InfoText.Mesh as TextMesh; _subText = SubText.Mesh as TextMesh; // Initialize eye tracking EyeTrackingPreferences.ReadSettingsFile(); InitializeEyeTracking(); GD.Print("Scene ready."); } private MeshInstance3D CreateCursor(Color color, string name) { var cursor = new MeshInstance3D(); cursor.Name = name; // Use BoxMesh instead of SphereMesh to avoid mesh generation issues var boxMesh = new SphereMesh(); boxMesh.Radius = CursorSize; boxMesh.Height = CursorSize * 2; // Create material with emission var material = new StandardMaterial3D(); material.AlbedoColor = color; material.ShadingMode = BaseMaterial3D.ShadingModeEnum.Unshaded; material.Transparency = BaseMaterial3D.TransparencyEnum.Alpha; // Add emission to make cursors glow and more visible material.EmissionEnabled = true; material.Emission = new Color(color.R, color.G, color.B, 1.0f); material.EmissionEnergyMultiplier = 3.0f; // Increased for better visibility // Assign mesh to instance cursor.Mesh = boxMesh; // Apply material override cursor.MaterialOverride = material; // Set initial position (straight ahead) so cursors are visible before gaze data arrives cursor.Position = new Vector3(0, 0, -CursorDistance); AddChild(cursor); GD.Print($"Created cursor '{name}' (Box) at position {cursor.Position} with size {CursorSize * 2}"); return cursor; } private void SetRingActiveVis(RingState ring, bool active) { var mat = ring.ringInstance.GetActiveMaterial(0) as StandardMaterial3D; if (mat != null) { mat.AlbedoColor = active ? ring.activeColor : ring.inactiveColor; InnerRing.SetSurfaceOverrideMaterial(0, mat); } } private async void InitializeEyeTracking() { try { // Create gaze estimation system _gazeEstimation = new GazeEstimation(); AddChild(_gazeEstimation); // Initialize camera bool success = await _gazeEstimation.cameraManager.InitializeCamera(); if (success) { // Load model if available var etSettings = ET_Preferences.Instance; if (etSettings != null && !string.IsNullOrEmpty(etSettings.selected_model)) { string modelPath = etSettings.selected_model; // Check if model path is an S3 URL if (modelPath.StartsWith("s3://")) { // Try to find a local downloaded version bool foundLocal = false; if (etSettings.downloaded_models != null && etSettings.downloaded_models.Count > 0) { // Use the first downloaded model as fallback modelPath = etSettings.downloaded_models[0]; foundLocal = true; GD.Print($"S3 model detected, using local model: {modelPath}"); } if (!foundLocal) { GD.PrintErr("Selected model is remote that hasn't beed downloaded.\nPlease download the model in the main app first."); UpdateInfoLabel("Model not downloaded!\nPlease download in main app", new Color(1, 0, 0)); modelPath = null; } } if (!string.IsNullOrEmpty(modelPath)) { GD.Print($"Loading model: {modelPath}"); _gazeEstimation.LoadModel(modelPath); } } else { GD.PrintErr("No model selected! Please select a model in the main application first."); UpdateInfoLabel("No model selected!\nPlease select model in main app", new Color(1, 0, 0)); } // Create test consumer _alignConsumer = new AlignmentConsumer(); _alignConsumer.XrCamera = _xrCamera; // Assign XR camera for head tracking _alignConsumer.RawGazeUpdated += OnRawGazeUpdated; AddChild(_alignConsumer); // Explicitly enable the consumer _alignConsumer.Enable(); // Register consumer with gaze estimation _gazeEstimation.RegisterObserver(_alignConsumer); GD.Print($"Alignment consumer initialized and registered (Enabled: {_alignConsumer.Enabled})"); GD.Print($"GazeEstimation IsRunning: {_gazeEstimation.IsRunning}"); UpdateInfoLabel(alignInstruction); } else { GD.PrintErr("Failed to initialize camera"); UpdateInfoLabel("Unable to initialize eyetracking cameras.\nEnable camera access in Windows settings and close any app that is using the camera.", new Color(1, 0, 0)); } } catch (Exception ex) { GD.PrintErr($"Error initializing eye tracking: {ex.Message}"); GD.PrintErr(ex.StackTrace); UpdateInfoLabel($"Error when initializing: {ex.Message}", new Color(1, 0, 0)); } } private void OnRawGazeUpdated(Vector3 leftVector, Vector3 rightVector, Vector3 combinedVector) { _lastRawCombined = combinedVector; } private void UpdateTarget(Vector3 gazeCoordinates) { if (quitPending) { return; } float rangeSq = Mathf.Abs(gazeCoordinates.X * gazeCoordinates.X + gazeCoordinates.Y * gazeCoordinates.Y); allRingsActive = true; // Check if gaze is within targets for (int i=0; i < target_rings.Count; i++) { RingState ring = target_rings[i]; ring.active = rangeSq < ring.targetThreshold; if (!ring.active) { allRingsActive = false; } // Activate ring if (ring.active && !ring.previouslyActive) { ring.previouslyActive = true; ring.hitSfx.Play(); } if (!ring.active && ring.previouslyActive) { ring.previouslyActive = false; } target_rings[i] = ring; SetRingActiveVis(ring, ring.active); } // Handle closing if the cursor remains in the middle long enough if (allRingsActive && !startedCloseProgress) { startedCloseProgress = true; if (closingCancellation != null) { closingCancellation.Cancel(); } BeginCloseProgress(); } if (!allRingsActive && !quitPending) { startedCloseProgress = false; closingInProgress = false; ProgCircle.Progress = 0; UpdateSubTextLabel(""); if (closingCancellation != null) { closingCancellation.Cancel(); } } } private async void BeginCloseProgress() { elapsed = 0; closeProgress = 0; closingCancellation = new CancellationTokenSource(); CancellationTokenSource currentToken = closingCancellation; await ToSignal(GetTree().CreateTimer(closeBeginDelay), "timeout"); currentToken.Token.ThrowIfCancellationRequested(); closingInProgress = true; } private async void QuitDelay() { var parentContainer = ProgCircle.GetParent() as MeshInstance3D; parentContainer.Visible = false; ProgCircle.Progress = 0; UpdateSubTextLabel("Alignment complete! Returning to SteamVR.."); await ToSignal(GetTree().CreateTimer(4), "timeout"); GetTree().Quit(); } float elapsed = 0; public override void _Process(double delta) { base._Process(delta); // Update circle visual during close if (closingInProgress && !quitPending) { if (closeProgress > 0.1) { UpdateSubTextLabel(closeInstruction); UpdateInfoLabel(""); } elapsed += (float)delta; closeProgress = Mathf.Clamp(elapsed / closeDuration, 0f, 1f); ProgCircle.Progress = closeProgress; if (closeProgress >= 1 && !quitPending) { quitPending = true; QuitDelay(); } } // Project cursors along 3D gaze direction, CursorDistance meters from the camera if (_xrCamera != null) { Vector3 cameraGlobalPos = _xrCamera.GlobalPosition; Basis cameraBasis = _xrCamera.GlobalTransform.Basis; // Raw cursor from raw combined gaze vector if (_rawCursor != null && _lastRawCombined != Vector3.Zero) { Vector3 localDir = _lastRawCombined.Normalized(); Vector3 worldDir = cameraBasis * localDir; _rawCursor.GlobalPosition = cameraGlobalPos + worldDir * CursorDistance; UpdateTarget(localDir); } } } private void UpdateInfoLabel(string text, Color? color = null) { if (_infoText != null) { InfoText.Visible = true; _infoText.Text = text; if (color.HasValue && InfoText.GetActiveMaterial(0) != null) { var material = InfoText.GetActiveMaterial(0) as StandardMaterial3D; material.AlbedoColor = color.Value; } } } private void UpdateSubTextLabel(string text, Color? color = null) { SubText.Visible = true; if (_subText != null) { _subText.Text = text; } if (color.HasValue && InfoText.GetActiveMaterial(0) != null) { var material = InfoText.GetActiveMaterial(0) as StandardMaterial3D; material.AlbedoColor = color.Value; } } public override void _ExitTree() { base._ExitTree(); // Clean up if (_alignConsumer != null) { _alignConsumer.RawGazeUpdated -= OnRawGazeUpdated; if (_gazeEstimation != null) { _gazeEstimation.RemoveObserver(_alignConsumer); } } GD.Print("Alignment scene cleaned up"); } }