using Godot; using System.Collections.Generic; using System.Diagnostics; using System.Threading; using OpenCvSharp; using ETPreferences; public partial class GazeEstimation : EyeTrackingProducer { // Reference to the neural network public NeuralNetwork _neuralNetwork { get; private set; } // Reference to the CameraManager public CameraManager cameraManager { get; private set; } public bool IsRunning { get { return _processingThread != null && _processingThread.IsAlive && cameraManager != null && cameraManager.IsInitialized && _neuralNetwork != null && !string.IsNullOrEmpty(_neuralNetwork.EncryptedModelPath); } } private Thread _processingThread; private CancellationTokenSource _cts; // Reusable Mat objects to avoid allocations private Mat _leftEyeImage; private Mat _rightEyeImage; // Processing optimization [Export] public int ProcessingThreadPriority { get; set; } = (int)ThreadPriority.AboveNormal; [Export] public int SleepOnEmptyFrame { get; set; } = 1; // ms to sleep when no frame is available [Export] public int SleepOnError { get; set; } = 33; // ms to sleep after error // Flag detection threshold for multi-task model [Export] public float FlagDetectionThreshold { get; set; } = 0.5f; // Benchmark settings [Export] public int BenchmarkIterations { get; set; } = 100; [Export] public bool RunBenchmark { get; set; } = true; // Called when the node enters the scene tree for the first time public override void _Ready() { // Create and initialize CameraManager cameraManager = new CameraManager(); cameraManager.DesiredWidth = 800; cameraManager.DesiredHeight = 400; cameraManager.FrameInterval = 0; // Process frames as fast as possible cameraManager.CameraBufferSize = 1; // Minimum buffer for lowest latency AddChild(cameraManager); GD.Print("Created new CameraManager node"); // Create and initialize neural network _neuralNetwork = new NeuralNetwork(); AddChild(_neuralNetwork); GD.Print("Created new NeuralNetwork node"); // Initialize reusable Mats _leftEyeImage = new Mat(); _rightEyeImage = new Mat(); // Start the processing thread StartProcessingThread(); GD.Print("GazeEstimation initialized successfully"); } private void StartProcessingThread() { _cts = new CancellationTokenSource(); _processingThread = new Thread(ProcessingLoop); _processingThread.Priority = (ThreadPriority)ProcessingThreadPriority; _processingThread.Start(); } private void ProcessingLoop() { // Preallocate Dictionary to avoid repeated allocations var predictionResults = new Dictionary(); while (!_cts.IsCancellationRequested) { Mat frame = null; try { // User configureable var trackRightEye = ET_Preferences.Instance != null ? ET_Preferences.Instance.right_eye_enabled : true; var trackLeftEye = ET_Preferences.Instance != null ? ET_Preferences.Instance.left_eye_enabled : true; // Get a frame from the camera frame = cameraManager.GetCameraFrame(); if (frame == null || frame.Empty()) { if (SleepOnEmptyFrame > 0) Thread.Sleep(SleepOnEmptyFrame); continue; } // Extract eye regions without creating new Mat objects // Use ROI (region of interest) instead of creating new Mat objects var leftROI = new Rect(0, 0, frame.Width / 2, frame.Height); var rightROI = new Rect(frame.Width / 2, 0, frame.Width / 2, frame.Height); frame[leftROI].CopyTo(_leftEyeImage); frame[rightROI].CopyTo(_rightEyeImage); // Process the image to get multi-task predictions MultiTaskPrediction leftEyePrediction; MultiTaskPrediction rightEyePrediction; // Process only one eye depending on the user's configuration if (trackRightEye && !trackLeftEye) { rightEyePrediction = _neuralNetwork.ProcessFrame(_rightEyeImage, true); leftEyePrediction = rightEyePrediction; } else if (trackLeftEye && !trackRightEye) { leftEyePrediction = _neuralNetwork.ProcessFrame(_leftEyeImage, false); rightEyePrediction = leftEyePrediction; } // Default to process both else { leftEyePrediction = _neuralNetwork.ProcessFrame(_leftEyeImage, false); // false = left eye rightEyePrediction = _neuralNetwork.ProcessFrame(_rightEyeImage, true); // true = right eye } // Create eye data and send to consumers if (leftEyePrediction != null && rightEyePrediction != null) { var eyeData = new EyeTrackingData { // Map position predictions to eye positions LeftEyeX = leftEyePrediction.Position.Length > 0 ? leftEyePrediction.Position[0] : 0.5f, LeftEyeY = leftEyePrediction.Position.Length > 1 ? leftEyePrediction.Position[1] : 0.5f, RightEyeX = rightEyePrediction.Position.Length > 0 ? rightEyePrediction.Position[0] : 0.5f, RightEyeY = rightEyePrediction.Position.Length > 1 ? rightEyePrediction.Position[1] : 0.5f, // Set flag probabilities LeftEyeFlagProbability = leftEyePrediction.FlagProbability, RightEyeFlagProbability = rightEyePrediction.FlagProbability, LeftEyeImage = _leftEyeImage.Clone(), RightEyeImage = _rightEyeImage.Clone(), Timestamp = (long)(Time.GetTicksMsec() / 1000.0f) }; // Set flag conditions based on threshold eyeData.SetFlagConditions(FlagDetectionThreshold); NotifyObservers(eyeData); } } catch (System.Exception ex) { GD.PrintErr($"Error in processing loop: {ex.Message}"); Thread.Sleep(SleepOnError); } finally { // Always dispose the frame to prevent memory leaks if (frame != null && !frame.IsDisposed) { frame.Dispose(); } } } } // Handle model selection from UI public void LoadModel(string modelPath) { GD.Print($"Model selected in GazeEstimation: {modelPath}"); // Update the neural network's model path and load it _neuralNetwork.EncryptedModelPath = modelPath; _neuralNetwork.LoadModel(); GD.Print("Neural network model loaded successfully"); } // Process a single frame and return multi-task predictions public (MultiTaskPrediction leftEye, MultiTaskPrediction rightEye) ProcessFrame(Mat left_eye_frame, Mat right_eye_frame) { var leftEyePrediction = _neuralNetwork.ProcessFrame(left_eye_frame, false); // false = left eye var rightEyePrediction = _neuralNetwork.ProcessFrame(right_eye_frame, true); // true = right eye return (leftEyePrediction, rightEyePrediction); } // Run a benchmark test on the provided image public void RunBenchmarkTest(Mat frame) { GD.Print($"Running benchmark with {BenchmarkIterations} iterations..."); // Create a duplicate of the image to avoid modifying the original var benchmarkImage = frame.Clone(); // Warm-up run (not measured) _neuralNetwork.ProcessFrame(benchmarkImage, false); // Start timing var stopwatch = new Stopwatch(); stopwatch.Start(); // Run the prediction multiple times for (int i = 0; i < BenchmarkIterations; i++) { _neuralNetwork.ProcessFrame(benchmarkImage, false); } // Stop timing stopwatch.Stop(); // Calculate and display performance metrics double totalSeconds = stopwatch.ElapsedMilliseconds / 1000.0; double averageMs = stopwatch.ElapsedMilliseconds / (double)BenchmarkIterations; double fps = BenchmarkIterations / totalSeconds; GD.Print("=== Performance Benchmark Results ==="); GD.Print($"Total time: {totalSeconds:F2} seconds"); GD.Print($"Average inference time: {averageMs:F2} ms per frame"); GD.Print($"Throughput: {fps:F2} FPS (frames per second)"); GD.Print("====================================="); } // Process a test image and optionally run a benchmark public void ProcessTestImage(string imagePath) { GD.Print($"Loading test image from: {imagePath}"); // Load the image resource var frame = Cv2.ImRead(imagePath); // Run a single prediction first to get the results var (leftEyePrediction, rightEyePrediction) = ProcessFrame(frame, frame); // Run benchmark if enabled if (RunBenchmark) { RunBenchmarkTest(frame); } } public override void _ExitTree() { // Clean up resources _cts?.Cancel(); _processingThread?.Join(1000); _leftEyeImage?.Dispose(); _rightEyeImage?.Dispose(); base._ExitTree(); } }