using Godot; using System; using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Security.Cryptography; using OpenCvSharp; using DirectShowLib; using HidSharp; /// /// Manages camera access and configuration for the application using OpenCvSharp. /// public partial class CameraManager : Node { [Export] public int DesiredWidth { get; set; } = 800; [Export] public int DesiredHeight { get; set; } = 400; [Export] public int MaxCameraIndex { get; set; } = 5; // Maximum camera index to check [Export] public int FrameInterval { get; set; } = 0; // Sleep time between frames in ms (0 = as fast as possible) [Export] public bool UseGrayscale { get; set; } = true; // Whether to convert frames to grayscale [Export] public int CameraBufferSize { get; set; } = 1; // Buffer size for camera frames (smaller = lower latency) [Export] public int MaxFailureAttempts { get; set; } = 3; // The amount of consecutive times the camera fails to capture a frame before a new search starts for the camera device [Export] public int MaxDuplicateFrameDectections { get; set; } = 5; // The amount of consecutive times a frame returns as duplicate as a previous one. Duplicate checks happen every 100 frames. private VideoCapture _videoCapture; private bool _isInitialized = false; private bool _searching = false; private bool _cameraAccessRevoked = false; private Thread _searchThread; private int captureFailureCount; private int foundDuplicateCount; private int duplicateDetectionTick; private string lastFrameHash = ""; public bool CameraAccessRevoked { get { return _cameraAccessRevoked; } } public bool IsInitialized { get { return _isInitialized; } } public bool Searching { get { return _searching; } } /// /// Called when the node enters the scene tree for the first time. /// public override void _Ready() { GD.Print("CameraManager: Ready"); } /// /// Searches for and sets up a camera with the desired resolution. /// /// True if a suitable camera was found and set up, false otherwise. private bool SetupCamera() { GD.Print("Searching for Bigeye camera..."); DsDevice[] devices = DsDevice.GetDevicesOfCat(FilterCategory.VideoInputDevice); int bigeyeIndex = -1; for (int i = 0; i < devices.Length; i++) { DsDevice device = devices[i]; GD.Print($"Found camera {i}: {device.Name}"); if (device.Name.Contains("Bigeye", StringComparison.OrdinalIgnoreCase)) { GD.Print($"Found Bigeye camera at index {i}"); bigeyeIndex = i; } } GD.Print($"Total cameras found: {devices.Length}"); // If Bigeye camera was found, select it using that index if (bigeyeIndex >= 0) { GD.Print($"Will select Bigeye camera at index {bigeyeIndex}"); } else { return false; } // Try to open the camera with DirectShow backend (equivalent to cv2.CAP_DSHOW in Python) _videoCapture = new VideoCapture(bigeyeIndex, VideoCaptureAPIs.DSHOW); if (_videoCapture.IsOpened()) { GD.Print($"Camera at index {bigeyeIndex} opened successfully"); _cameraAccessRevoked = false; } else { GD.Print($"Camera at index {bigeyeIndex} failed to open"); _cameraAccessRevoked = true; _videoCapture.Dispose(); _videoCapture = null; return false; } // Set camera resolution to desired dimensions _videoCapture.Set(VideoCaptureProperties.FrameWidth, DesiredWidth); _videoCapture.Set(VideoCaptureProperties.FrameHeight, DesiredHeight); // Optimize for latency: reduce buffer size _videoCapture.Set(VideoCaptureProperties.BufferSize, CameraBufferSize); // Verify the settings took effect int setWidth = (int)_videoCapture.Get(VideoCaptureProperties.FrameWidth); int setHeight = (int)_videoCapture.Get(VideoCaptureProperties.FrameHeight); if (setWidth != DesiredWidth || setHeight != DesiredHeight) { GD.PrintErr($"Failed to set desired resolution: {DesiredWidth}x{DesiredHeight}"); _videoCapture.Dispose(); _videoCapture = null; return false; } using var frame = new Mat(); if (_videoCapture.Read(frame) && !frame.Empty()) { GD.Print($"Set resolution to: {setWidth}x{setHeight}"); GD.Print($"Camera buffer size: {_videoCapture.Get(VideoCaptureProperties.BufferSize)}"); } else { GD.Print($"Could not read frames from camera at index {bigeyeIndex}"); _videoCapture.Dispose(); _videoCapture = null; return false; } return true; } /// /// Initializes the camera system. /// /// True if initialization was successful, false otherwise. public async Task InitializeCamera() { if (_isInitialized) { GD.Print("Camera already initialized"); return true; } GD.Print("Initializing camera..."); if (SetupCamera()) { _isInitialized = true; // Wait a moment for the camera to stabilize await Task.Delay(500); GD.Print("Camera initialized successfully"); return true; } GD.PrintErr("Failed to initialize camera"); return false; } /// /// Start camera search loop /// public void StartCameraSearch() { if (_searching) { GD.PrintErr("Not starting another search since one is already running."); return; } _searchThread = new Thread(FindCameraLoop); _searchThread.Start(); } /// /// Keeps attempting to find the eyetracking cameras /// private async void FindCameraLoop() { if (_videoCapture != null || _searching) { GD.PrintErr("Not starting a search for eyetracking cameras as a previous instance has not been properly released."); return; } _searching = true; while (_searching) { try { GD.Print("Attempting to find eyetracking cameras"); bool success = await InitializeCamera(); if (success) { _searching = false; return; } else { GD.PrintErr("Failed to find eyetracking cameras. Trying again..."); } } catch (Exception ex) { GD.PrintErr($"Failed to find eyetracking cameras: {ex.Message}"); } Thread.Sleep(1000); } } string GetFrameHash(Mat frame) { var buffer = frame.ImEncode(".jpg"); using var sha256 = SHA256.Create(); var hashBytes = sha256.ComputeHash(buffer); return Convert.ToBase64String(hashBytes); } /// /// Gets the latest frame from the webcam as an OpenCV Mat. /// /// The latest camera frame as a Mat, or null if no frame is available. public Mat GetCameraFrame() { if (!_isInitialized || _videoCapture == null || !_videoCapture.IsOpened()) { return null; } try { // Direct capture approach - no thread, no clone var frame = new Mat(); if (_videoCapture.Grab() && _videoCapture.Retrieve(frame) && !frame.Empty()) { captureFailureCount = 0; // Convert to grayscale only if needed if (UseGrayscale && frame.Channels() > 1) { Cv2.CvtColor(frame, frame, ColorConversionCodes.BGR2GRAY); } // Check corner 10x10 pixels and reset camera if they sum to 0 if (frame.Width >= 10 && frame.Height >= 10) { var cornerRegion = frame[new Rect(0, 0, 10, 10)]; var sum = Cv2.Sum(cornerRegion); bool allZero = sum.Val0 == 0; if (frame.Channels() > 1) { allZero = allZero && sum.Val1 == 0 && sum.Val2 == 0; if (frame.Channels() > 3) { allZero = allZero && sum.Val3 == 0; } } if (allZero) { GD.Print("Corner 10x10 pixels sum to 0, resetting camera"); ResetCamera(); throw new FormatException("Captured frames have invalid format."); } } // Finding duplicate frames // System sleep / hibernate can cause a frozen capture stream duplicateDetectionTick++; if (duplicateDetectionTick % 100 == 0) { duplicateDetectionTick = 0; string newHash = GetFrameHash(frame); if (newHash == lastFrameHash) { foundDuplicateCount++; if (foundDuplicateCount >= MaxDuplicateFrameDectections) { foundDuplicateCount = 0; ReleaseCamera(); StartCameraSearch(); throw new FormatException("Too many duplicate frames! Reinitializing cameras..."); } } else { foundDuplicateCount = 0; } lastFrameHash = newHash; } return frame; } else { GD.PrintErr("Failed to read frame from camera"); // After X amount of consecutive failures, search for cameras again captureFailureCount++; if (captureFailureCount >= MaxFailureAttempts) { captureFailureCount = 0; ReleaseCamera(); StartCameraSearch(); throw new FormatException("Could not read frames: camera disconnected."); } frame.Dispose(); return null; } } catch (Exception ex) { GD.PrintErr($"Error getting camera frame: {ex.Message}"); throw; } } /// /// Releases camera resources. /// public void ReleaseCamera() { // Release OpenCV resources if (_videoCapture != null) { if (_videoCapture.IsOpened()) { _videoCapture.Release(); } _videoCapture.Dispose(); _videoCapture = null; } // Stop searching if (_searching) { _searching = false; _searchThread?.Join(1000); _searchThread = null; } _isInitialized = false; GD.Print("Camera released"); } /// /// Called when the node is about to be removed from the scene. /// public override void _ExitTree() { ReleaseCamera(); base._ExitTree(); } /// /// Resets the camera by sending HID command to the eyetracking device. /// /// True if the reset command was sent successfully, false otherwise. public bool ResetCamera() { try { // Device VID and PID const int VID = 0x35bd; const int PID = 0x0101; GD.Print($"Attempting to reset camera via HID command to device VID:{VID:X4} PID:{PID:X4}"); // Find the HID device var deviceList = DeviceList.Local; var hidDevice = deviceList.GetHidDevices(VID, PID).FirstOrDefault(); if (hidDevice == null) { GD.PrintErr($"HID device with VID:{VID:X4} PID:{PID:X4} not found"); return false; } // Command bytes: [0, ord('e'), ord('R')] = [0, 101, 82] byte[] commandBytes = new byte[hidDevice.GetMaxFeatureReportLength()]; commandBytes[0] = 0; commandBytes[1] = 101; commandBytes[2] = 82; GD.Print($"Found HID device: {hidDevice.GetFriendlyName()}"); // Open the device and send the command using (var stream = hidDevice.Open()) { if (stream.CanWrite) { stream.SetFeature(commandBytes); GD.Print("Camera reset command sent successfully"); return true; } else { GD.PrintErr("Cannot write to HID device"); return false; } } } catch (Exception ex) { GD.PrintErr($"Error sending camera reset command: {ex.Message}"); return false; } } }