#define WIN32_LEAN_AND_MEAN #define NOMINMAX #define _WINSOCKAPI_ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "eyetracking_runtime.h" #include "ui.h" #include "utils.h" #include "json11.hpp" #include #include "steamvr_watch.h" #include #pragma comment(lib, "strmiids.lib") #include "onnxruntime_c_api.h" using namespace json11; namespace { void AppendRuntimeLog(const std::string& message); struct MultiTaskPrediction; struct EyeTrackingData; class FrustumLoader; class EtInference; class MemmapPublisher; class VRCFTPublisher; class VRChatPublisher; enum class EtBlinkMode : int { Blinking = 0, Winking = 1, None = 2, }; EtBlinkMode NormalizeBlinkMode(int raw_mode) { switch (raw_mode) { case static_cast(EtBlinkMode::Winking): return EtBlinkMode::Winking; case static_cast(EtBlinkMode::None): return EtBlinkMode::None; default: return EtBlinkMode::Blinking; } } static std::atomic g_last_seh_code{ 0 }; static std::atomic g_seh_count{ 0 }; void LogOnce(const std::string& message, int& counter, int limit = 5) { if (counter < limit) { AppendRuntimeLog(message); counter++; } } double GetTicksMsec() { static const auto start = std::chrono::steady_clock::now(); return std::chrono::duration( std::chrono::steady_clock::now() - start ).count(); } OrtStatus* RunOrtWithSeh(const OrtApi* api, OrtSession* session, const char* const* input_names, const OrtValue* const* input_values, size_t input_count, const char* const* output_names, size_t output_count, OrtValue** output_values, bool* seh_hit) { if (seh_hit) { *seh_hit = false; } __try { return api->Run(session, nullptr, input_names, input_values, input_count, output_names, output_count, output_values); } __except (EXCEPTION_EXECUTE_HANDLER) { if (seh_hit) { *seh_hit = true; } return nullptr; } } bool TryAppendDmlWithSeh(OrtStatus* (ORT_API_CALL* fn)(OrtSessionOptions*, int), OrtSessionOptions* options, OrtStatus** out_status, bool* seh_hit) { if (seh_hit) { *seh_hit = false; } if (out_status) { *out_status = nullptr; } __try { if (out_status) { *out_status = fn(options, 0); } else { fn(options, 0); } return true; } __except (EXCEPTION_EXECUTE_HANDLER) { if (seh_hit) { *seh_hit = true; } return false; } } bool TryCreateSessionFromArrayWithSeh(const OrtApi* api, OrtEnv* env, const void* model_data, size_t model_size, OrtSessionOptions* options, OrtSession** out_session, OrtStatus** out_status, bool* seh_hit) { if (seh_hit) { *seh_hit = false; } if (out_status) { *out_status = nullptr; } __try { if (out_status) { *out_status = api->CreateSessionFromArray(env, model_data, model_size, options, out_session); } else { api->CreateSessionFromArray(env, model_data, model_size, options, out_session); } return true; } __except (EXCEPTION_EXECUTE_HANDLER) { if (seh_hit) { *seh_hit = true; } return false; } } bool RunEyeWithSeh(EtInference* inference, const cv::Mat* eye, bool is_right, MultiTaskPrediction* out, bool* seh_hit); bool MemmapProcessWithSeh(MemmapPublisher* memmap, const EyeTrackingData* data, const FrustumLoader* frustum, bool* seh_hit); bool VrcftOnDataWithSeh(VRCFTPublisher* vrcft, const EyeTrackingData* data, bool* seh_hit); bool VrchatOnDataWithSeh(VRChatPublisher* vrchat, const EyeTrackingData* data, bool* seh_hit); // MSVC C++ exception code (throw); often raised/caught inside ONNX/DirectML during session create. static constexpr uint32_t kMsvcCxxExceptionCode = 0xe06d7363u; LONG WINAPI EtRuntimeExceptionHandler(PEXCEPTION_POINTERS info) { if (utils.is_file_dialog_active()) { return EXCEPTION_CONTINUE_SEARCH; } const uint32_t seh_code = static_cast(info->ExceptionRecord->ExceptionCode); g_last_seh_code.store(seh_code); const int count = g_seh_count.fetch_add(1); if (seh_code == kMsvcCxxExceptionCode) { if (count < 2) { std::ostringstream msg; msg << "SEH exception: 0x" << std::hex << seh_code << " (C++ exception from dependency; may be handled internally) at 0x" << info->ExceptionRecord->ExceptionAddress; AppendRuntimeLog(msg.str()); } else if (count == 2) { AppendRuntimeLog("SEH: further 0xe06d7363 (C++ exception) messages suppressed to avoid log flood."); } return EXCEPTION_CONTINUE_SEARCH; } std::ostringstream msg; msg << "SEH exception: 0x" << std::hex << info->ExceptionRecord->ExceptionCode << " at 0x" << info->ExceptionRecord->ExceptionAddress; AppendRuntimeLog(msg.str()); return EXCEPTION_CONTINUE_SEARCH; } void ClearRuntimeLog() { if (utils.is_file_dialog_active()) { return; } try { std::ofstream out("eyetracking_runtime.log", std::ios::trunc); out.close(); } catch (...) {} } void AppendRuntimeLog(const std::string& message) { if (utils.is_file_dialog_active()) { return; } try { std::ofstream out("eyetracking_runtime.log", std::ios::app); if (!out.is_open()) { return; } auto now = std::chrono::system_clock::now(); std::time_t t = std::chrono::system_clock::to_time_t(now); std::tm tm{}; localtime_s(&tm, &t); out << std::put_time(&tm, "%Y-%m-%d %H:%M:%S") << " | " << message << "\n"; } catch (...) { } } struct Vec2 { float x{}; float y{}; }; struct Vec3 { float x{}; float y{}; float z{}; }; struct Vec4 { float x{}; float y{}; float z{}; float w{}; }; struct Mat4 { float m[4][4]{}; static Mat4 FromColumns(const Vec4& x, const Vec4& y, const Vec4& z, const Vec4& w) { Mat4 out{}; out.m[0][0] = x.x; out.m[1][0] = x.y; out.m[2][0] = x.z; out.m[3][0] = x.w; out.m[0][1] = y.x; out.m[1][1] = y.y; out.m[2][1] = y.z; out.m[3][1] = y.w; out.m[0][2] = z.x; out.m[1][2] = z.y; out.m[2][2] = z.z; out.m[3][2] = z.w; out.m[0][3] = w.x; out.m[1][3] = w.y; out.m[2][3] = w.z; out.m[3][3] = w.w; return out; } Vec4 Mul(const Vec4& v) const { Vec4 out{}; out.x = m[0][0] * v.x + m[0][1] * v.y + m[0][2] * v.z + m[0][3] * v.w; out.y = m[1][0] * v.x + m[1][1] * v.y + m[1][2] * v.z + m[1][3] * v.w; out.z = m[2][0] * v.x + m[2][1] * v.y + m[2][2] * v.z + m[2][3] * v.w; out.w = m[3][0] * v.x + m[3][1] * v.y + m[3][2] * v.z + m[3][3] * v.w; return out; } Mat4 Inverse() const { Mat4 inv{}; float invOut[16]{}; const float* a = &m[0][0]; invOut[0] = a[5] * a[10] * a[15] - a[5] * a[11] * a[14] - a[9] * a[6] * a[15] + a[9] * a[7] * a[14] + a[13] * a[6] * a[11] - a[13] * a[7] * a[10]; invOut[4] = -a[4] * a[10] * a[15] + a[4] * a[11] * a[14] + a[8] * a[6] * a[15] - a[8] * a[7] * a[14] - a[12] * a[6] * a[11] + a[12] * a[7] * a[10]; invOut[8] = a[4] * a[9] * a[15] - a[4] * a[11] * a[13] - a[8] * a[5] * a[15] + a[8] * a[7] * a[13] + a[12] * a[5] * a[11] - a[12] * a[7] * a[9]; invOut[12] = -a[4] * a[9] * a[14] + a[4] * a[10] * a[13] + a[8] * a[5] * a[14] - a[8] * a[6] * a[13] - a[12] * a[5] * a[10] + a[12] * a[6] * a[9]; invOut[1] = -a[1] * a[10] * a[15] + a[1] * a[11] * a[14] + a[9] * a[2] * a[15] - a[9] * a[3] * a[14] - a[13] * a[2] * a[11] + a[13] * a[3] * a[10]; invOut[5] = a[0] * a[10] * a[15] - a[0] * a[11] * a[14] - a[8] * a[2] * a[15] + a[8] * a[3] * a[14] + a[12] * a[2] * a[11] - a[12] * a[3] * a[10]; invOut[9] = -a[0] * a[9] * a[15] + a[0] * a[11] * a[13] + a[8] * a[1] * a[15] - a[8] * a[3] * a[13] - a[12] * a[1] * a[11] + a[12] * a[3] * a[9]; invOut[13] = a[0] * a[9] * a[14] - a[0] * a[10] * a[13] - a[8] * a[1] * a[14] + a[8] * a[2] * a[13] + a[12] * a[1] * a[10] - a[12] * a[2] * a[9]; invOut[2] = a[1] * a[6] * a[15] - a[1] * a[7] * a[14] - a[5] * a[2] * a[15] + a[5] * a[3] * a[14] + a[13] * a[2] * a[7] - a[13] * a[3] * a[6]; invOut[6] = -a[0] * a[6] * a[15] + a[0] * a[7] * a[14] + a[4] * a[2] * a[15] - a[4] * a[3] * a[14] - a[12] * a[2] * a[7] + a[12] * a[3] * a[6]; invOut[10] = a[0] * a[5] * a[15] - a[0] * a[7] * a[13] - a[4] * a[1] * a[15] + a[4] * a[3] * a[13] + a[12] * a[1] * a[7] - a[12] * a[3] * a[5]; invOut[14] = -a[0] * a[5] * a[14] + a[0] * a[6] * a[13] + a[4] * a[1] * a[14] - a[4] * a[2] * a[13] - a[12] * a[1] * a[6] + a[12] * a[2] * a[5]; invOut[3] = -a[1] * a[6] * a[11] + a[1] * a[7] * a[10] + a[5] * a[2] * a[11] - a[5] * a[3] * a[10] - a[9] * a[2] * a[7] + a[9] * a[3] * a[6]; invOut[7] = a[0] * a[6] * a[11] - a[0] * a[7] * a[10] - a[4] * a[2] * a[11] + a[4] * a[3] * a[10] + a[8] * a[2] * a[7] - a[8] * a[3] * a[6]; invOut[11] = -a[0] * a[5] * a[11] + a[0] * a[7] * a[9] + a[4] * a[1] * a[11] - a[4] * a[3] * a[9] - a[8] * a[1] * a[7] + a[8] * a[3] * a[5]; invOut[15] = a[0] * a[5] * a[10] - a[0] * a[6] * a[9] - a[4] * a[1] * a[10] + a[4] * a[2] * a[9] + a[8] * a[1] * a[6] - a[8] * a[2] * a[5]; float det = a[0] * invOut[0] + a[1] * invOut[4] + a[2] * invOut[8] + a[3] * invOut[12]; if (std::fabs(det) < 1e-8f) { return Mat4{}; } det = 1.0f / det; for (int i = 0; i < 16; i++) { invOut[i] *= det; } for (int r = 0; r < 4; r++) { for (int c = 0; c < 4; c++) { inv.m[r][c] = invOut[r * 4 + c]; } } return inv; } }; struct EyeTrackingData { float left_x = 0.5f; float left_y = 0.5f; float right_x = 0.5f; float right_y = 0.5f; float left_openness = 1.0f; float right_openness = 1.0f; float confidence = 0.8f; float left_flag_probability = 0.0f; float right_flag_probability = 0.0f; bool left_flag_condition = false; bool right_flag_condition = false; int64_t timestamp_ms = 0; cv::Mat left_eye_image; cv::Mat right_eye_image; bool HasAnyFlagCondition() const { return left_flag_condition || right_flag_condition; } void SetFlagConditions(float threshold) { left_flag_condition = left_flag_probability > threshold; right_flag_condition = right_flag_probability > threshold; } }; struct MultiTaskPrediction { std::array position{ 0.5f, 0.5f }; float flag_probability = 0.0f; }; class OneEuroFilter { public: OneEuroFilter(float minCutoff = 1.0f, float beta = 0.0f, float dCutoff = 1.0f) : min_cutoff(minCutoff), beta(beta), d_cutoff(dCutoff) {} float Filter(float value, double time) { if (first_time) { last_time = time; last_value = value; last_delta = 0.0f; first_time = false; return value; } if (time <= last_time) return last_value; float delta = (value - last_value) / (time - last_time); float filtered_delta = Alpha(d_cutoff, time - last_time) * delta + (1.0f - Alpha(d_cutoff, time - last_time)) * last_delta; float cutoff = min_cutoff + beta * std::fabs(filtered_delta); float filtered_value = Alpha(cutoff, time - last_time) * value + (1.0f - Alpha(cutoff, time - last_time)) * last_value; last_time = time; last_value = filtered_value; last_delta = filtered_delta; return filtered_value; } void Reset() { first_time = true; } void SetMinCutoff(float v) { min_cutoff = v; } void SetBeta(float v) { beta = v; } void SetDCutoff(float v) { d_cutoff = v; } private: float Alpha(float cutoff, float delta_time) { float tau = 1.0f / (2.0f * 3.14159265f * cutoff); return 1.0f / (1.0f + tau / delta_time); } float min_cutoff = 1.0f; float beta = 0.0f; float d_cutoff = 1.0f; float last_time = 0.0f; float last_value = 0.0f; float last_delta = 0.0f; bool first_time = true; }; class BlinkLerp { public: BlinkLerp(float duration = 0.05f) : duration(std::max(duration, 0.001f)) {} float UpdateBlinkLerp(bool blinking, float current_time, float base_closed) { return UpdateOneEye(blinking, current_time, base_closed, was_blinking, blink_start, lerping, opening); } std::pair UpdateBlinkLerpPerEye(bool left_blink, bool right_blink, float current_time, float left_base, float right_base) { float left_closed = UpdateOneEye(left_blink, current_time, left_base, was_left_blink, left_start, left_lerping, left_opening); float right_closed = UpdateOneEye(right_blink, current_time, right_base, was_right_blink, right_start, right_lerping, right_opening); const float fully_open_eps = 0.0001f; bool left_open = left_opening && left_lerping; bool right_open = right_opening && right_lerping; if (left_open && right_open) { float sync_amount = std::min(left_closed, right_closed); left_closed = right_closed = sync_amount; } else if (left_open && right_closed > fully_open_eps) { right_closed = left_closed; } else if (right_open && left_closed > fully_open_eps) { left_closed = right_closed; } return { left_closed, right_closed }; } void Reset() { was_blinking = false; lerping = false; opening = false; was_left_blink = false; was_right_blink = false; left_lerping = false; right_lerping = false; left_opening = false; right_opening = false; } bool IsLerping() const { return lerping || left_lerping || right_lerping; } private: float UpdateOneEye(bool blinking, float current_time, float base_closed, bool& was_blink, float& start_time, bool& is_lerping, bool& is_opening) { bool into = !was_blink && blinking; bool out = was_blink && !blinking; if (into) { start_time = current_time; is_lerping = true; is_opening = false; } else if (out) { start_time = current_time; is_lerping = true; is_opening = true; } float closed = base_closed; if (is_lerping) { float elapsed = current_time - start_time; float progress = std::min(elapsed / duration, 1.0f); closed = is_opening ? (1.0f - progress) : progress; if (progress >= 1.0f) { is_lerping = false; } } else if (blinking) { closed = 1.0f; } was_blink = blinking; return std::clamp(closed, 0.0f, 1.0f); } float duration = 0.05f; float blink_start = 0.0f; bool was_blinking = false; bool lerping = false; bool opening = false; bool was_left_blink = false; bool was_right_blink = false; float left_start = 0.0f; float right_start = 0.0f; bool left_lerping = false; bool right_lerping = false; bool left_opening = false; bool right_opening = false; }; class EyeTrackingBuffer { public: EyeTrackingBuffer(int max_size, int min_size, std::function&)> analyze, std::function process, std::function can_process) : max_size(max_size), min_size(min_size), analyze_chunk(std::move(analyze)), process_data(std::move(process)), can_process(std::move(can_process)) {} void AddData(const EyeTrackingData& data) { { std::lock_guard guard(lock); buffer.push_back(data); while (static_cast(buffer.size()) > max_size) { buffer.erase(buffer.begin()); } } ProcessBuffer(); } void Clear() { std::lock_guard guard(lock); buffer.clear(); } private: void ProcessBuffer() { if (!can_process()) { return; } bool should_process = false; size_t buffer_count = 0; { std::lock_guard guard(lock); buffer_count = buffer.size(); should_process = buffer_count >= static_cast(min_size); } if (!should_process) { return; } EyeTrackingData data_to_process{}; std::vector current_buffer; { std::lock_guard guard(lock); if (!buffer.empty()) { data_to_process = buffer.front(); buffer.erase(buffer.begin()); current_buffer = buffer; } } if (!current_buffer.empty() || buffer_count > 0) { current_buffer.insert(current_buffer.begin(), data_to_process); EyeTrackingData processed = analyze_chunk(current_buffer); process_data(processed); } } std::vector buffer; std::mutex lock; int max_size = 40; int min_size = 10; std::function&)> analyze_chunk; std::function process_data; std::function can_process; }; class EyeConvergence { public: static std::pair Apply(const Vec2& left, const Vec2& right, float strength = 0.5f) { Vec2 average{ (left.x + right.x) * 0.5f, (left.y + right.y) * 0.5f }; Vec2 left_dev{ left.x - average.x, left.y - average.y }; Vec2 right_dev{ right.x - average.x, right.y - average.y }; bool cross = (left.x > 0.1f && right.x < -0.1f); float eff = cross ? strength * 0.3f : strength; Vec2 left_conv{ average.x + left_dev.x * (1.0f - eff), average.y + left_dev.y * (1.0f - eff) }; Vec2 right_conv{ average.x + right_dev.x * (1.0f - eff), average.y + right_dev.y * (1.0f - eff) }; return { left_conv, right_conv }; } }; class FrustumLoader { public: bool Load(std::string& error) { // Find the frustum data from relative folder std::filesystem::path path = local_path / target_file; if (!std::filesystem::exists(path)) { // Fallback to legacy client folder path = legacy_path / target_file; } std::ifstream in(path); if (!in.is_open()) { error = "Frustum data file not found."; loaded = false; return false; } std::stringstream buffer; buffer << in.rdbuf(); std::string err; Json json = Json::parse(buffer.str(), err); if (!err.empty()) { error = "Failed to parse frustum JSON."; loaded = false; return false; } auto left_proj = json["LeftProjection"]; auto right_proj = json["RightProjection"]; if (!left_proj.is_object() || !right_proj.is_object()) { error = "Invalid frustum JSON layout."; loaded = false; return false; } Mat4 left = ParseProjection(left_proj); Mat4 right = ParseProjection(right_proj); left_proj_matrix = left; right_proj_matrix = right; inv_left = left.Inverse(); inv_right = right.Inverse(); loaded = true; return true; } bool IsLoaded() const { return loaded; } Vec2 ScreenPositionToViewAngles(const Vec2& screen_pos, bool use_left) const { Vec2 ndc{ screen_pos.x * 2.0f - 1.0f, screen_pos.y * 2.0f - 1.0f }; Mat4 inv = use_left ? inv_left : inv_right; Vec4 eye = inv.Mul(Vec4{ ndc.x, ndc.y, 1.0f, 1.0f }); if (std::fabs(eye.w) < 0.0001f) { return Vec2{}; } eye.x /= eye.w; eye.y /= eye.w; eye.z /= eye.w; Vec3 view{ eye.x, eye.y, eye.z }; float length = std::sqrt(view.x * view.x + view.y * view.y + view.z * view.z); if (length > 0.0f) { view.x /= length; view.y /= length; view.z /= length; } float yaw = std::atan2(view.x, -view.z) * 180.0f / 3.14159265f; float pitch = std::atan2(view.y, std::sqrt(view.x * view.x + view.z * view.z)) * 180.0f / 3.14159265f; return Vec2{ yaw - (use_left ? 1.0f : -1.0f) * 3.085f, pitch }; } private: Mat4 ParseProjection(const Json& proj) const { auto x = proj["X"].array_items(); auto y = proj["Y"].array_items(); auto z = proj["Z"].array_items(); auto w = proj["W"].array_items(); Vec4 cx{ (float)x[0].number_value(), (float)x[1].number_value(), (float)x[2].number_value(), (float)x[3].number_value() }; Vec4 cy{ (float)y[0].number_value(), (float)y[1].number_value(), (float)y[2].number_value(), (float)y[3].number_value() }; Vec4 cz{ (float)z[0].number_value(), (float)z[1].number_value(), (float)z[2].number_value(), (float)z[3].number_value() }; Vec4 cw{ (float)w[0].number_value(), (float)w[1].number_value(), (float)w[2].number_value(), (float)w[3].number_value() }; return Mat4::FromColumns(cx, cy, cz, cw); } std::filesystem::path legacy_path = "eyetracking\\ETClient\\recorded_data"; std::filesystem::path local_path = "eyetracking\\ETEnroll\\recorded_data"; std::filesystem::path target_file = "frustum_data.json"; Mat4 left_proj_matrix{}; Mat4 right_proj_matrix{}; Mat4 inv_left{}; Mat4 inv_right{}; bool loaded = false; }; class OscClient { public: bool Initialize(const std::string& ip, int port, std::string& error) { Shutdown(); WSADATA wsaData; if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0) { error = "WSAStartup failed."; return false; } sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); if (sock == INVALID_SOCKET) { error = "Failed to create UDP socket."; return false; } sockaddr_in addr{}; addr.sin_family = AF_INET; addr.sin_port = htons(static_cast(port)); if (InetPtonA(AF_INET, ip.c_str(), &addr.sin_addr) != 1) { error = "Invalid OSC IP address."; return false; } target = addr; return true; } void Shutdown() { if (sock != INVALID_SOCKET) { closesocket(sock); sock = INVALID_SOCKET; } } bool SendMessage(const std::string& address, float value) { float values[1] = { value }; return SendMessage(address, values, 1); } bool SendMessage(const std::string& address, const float* values, size_t count) { if (sock == INVALID_SOCKET) { return false; } std::vector packet; AppendString(packet, address); std::string type = ","; type.append(count, 'f'); AppendString(packet, type); for (size_t i = 0; i < count; i++) { AppendFloat(packet, values[i]); } int result = sendto(sock, reinterpret_cast(packet.data()), static_cast(packet.size()), 0, reinterpret_cast(&target), sizeof(target)); return result != SOCKET_ERROR; } private: void AppendString(std::vector& buffer, const std::string& str) { buffer.insert(buffer.end(), str.begin(), str.end()); buffer.push_back(0); while (buffer.size() % 4 != 0) { buffer.push_back(0); } } void AppendFloat(std::vector& buffer, float value) { uint32_t data; static_assert(sizeof(float) == sizeof(uint32_t), "float size mismatch"); std::memcpy(&data, &value, sizeof(uint32_t)); data = htonl(data); // Send bytes in memory order (first byte at &data = MSB for network) buffer.push_back(static_cast(data & 0xFF)); buffer.push_back(static_cast((data >> 8) & 0xFF)); buffer.push_back(static_cast((data >> 16) & 0xFF)); buffer.push_back(static_cast((data >> 24) & 0xFF)); } SOCKET sock = INVALID_SOCKET; sockaddr_in target{}; }; class EtCamera { public: int FindBigeyeCameraIndex() { HRESULT hr = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED); bool coInit = SUCCEEDED(hr) || hr == RPC_E_CHANGED_MODE; int bigeyeIndex = -1; ICreateDevEnum* pDevEnum = nullptr; IEnumMoniker* pEnum = nullptr; hr = CoCreateInstance(CLSID_SystemDeviceEnum, nullptr, CLSCTX_INPROC_SERVER, IID_ICreateDevEnum, reinterpret_cast(&pDevEnum)); if (FAILED(hr) || !pDevEnum) { if (coInit) CoUninitialize(); return -1; } hr = pDevEnum->CreateClassEnumerator(CLSID_VideoInputDeviceCategory, &pEnum, 0); if (hr != S_OK || !pEnum) { pDevEnum->Release(); if (coInit) CoUninitialize(); return -1; } IMoniker* pMoniker = nullptr; ULONG fetched = 0; int index = 0; while (pEnum->Next(1, &pMoniker, &fetched) == S_OK) { IPropertyBag* pPropBag = nullptr; hr = pMoniker->BindToStorage(0, 0, IID_IPropertyBag, reinterpret_cast(&pPropBag)); if (SUCCEEDED(hr)) { VARIANT varName; VariantInit(&varName); hr = pPropBag->Read(L"FriendlyName", &varName, 0); if (SUCCEEDED(hr) && varName.vt == VT_BSTR) { std::wstring name(varName.bstrVal, SysStringLen(varName.bstrVal)); if (name.find(L"Bigeye") != std::wstring::npos) { bigeyeIndex = index; VariantClear(&varName); pPropBag->Release(); pMoniker->Release(); break; } } VariantClear(&varName); pPropBag->Release(); } pMoniker->Release(); ++index; } if (pEnum) pEnum->Release(); if (pDevEnum) pDevEnum->Release(); if (coInit) CoUninitialize(); return bigeyeIndex; } bool SetupCamera(int desired_width, int desired_height, int buffer_size) { AppendRuntimeLog("CameraManager: Searching for Bigeye camera..."); int bigeyeIndex = FindBigeyeCameraIndex(); if (bigeyeIndex < 0) { AppendRuntimeLog("CameraManager: Bigeye camera not found."); return false; } AppendRuntimeLog("CameraManager: Will select Bigeye camera at index " + std::to_string(bigeyeIndex)); // Open with DirectShow backend capture.open(bigeyeIndex, cv::CAP_DSHOW); if (!capture.isOpened()) { AppendRuntimeLog("CameraManager: Camera at index " + std::to_string(bigeyeIndex) + " failed to open"); eyetracking_runtime.SetCameraAccessRevoked(true); capture.release(); return false; } eyetracking_runtime.SetCameraAccessRevoked(false); AppendRuntimeLog("CameraManager: Camera at index " + std::to_string(bigeyeIndex)); // Set resolution and buffer size capture.set(cv::CAP_PROP_FRAME_WIDTH, desired_width); capture.set(cv::CAP_PROP_FRAME_HEIGHT, desired_height); capture.set(cv::CAP_PROP_BUFFERSIZE, buffer_size); int setWidth = static_cast(capture.get(cv::CAP_PROP_FRAME_WIDTH)); int setHeight = static_cast(capture.get(cv::CAP_PROP_FRAME_HEIGHT)); if (setWidth != desired_width || setHeight != desired_height) { AppendRuntimeLog("CameraManager: Failed to set desired resolution: " + std::to_string(desired_width) + "x" + std::to_string(desired_height)); capture.release(); return false; } cv::Mat frame; if (capture.read(frame) && !frame.empty()) { AppendRuntimeLog("CameraManager: Set resolution to: " + std::to_string(setWidth) + "x" + std::to_string(setHeight)); AppendRuntimeLog("CameraManager: Camera buffer size: " + std::to_string(capture.get(cv::CAP_PROP_BUFFERSIZE))); } else { AppendRuntimeLog("CameraManager: Could not read frames from camera at index: " + std::to_string(bigeyeIndex)); capture.release(); return false; } return true; } bool Initialize() { if (eyetracking_runtime.CameraInitialized()) { AppendRuntimeLog("CameraManager: Camera already initialized"); return true; } if (steamvrWatch.BlockFromSteamVRAbsent()) { AppendRuntimeLog("CameraManager: Not locking camera resource as SteamVR is not running"); return false; } AppendRuntimeLog("CameraManager: Initializing camera..."); std::lock_guard lock(camera_mutex); if (SetupCamera(eyetracking_runtime.DesiredWidth, eyetracking_runtime.DesiredHeight, eyetracking_runtime.CameraBufferSize)) { eyetracking_runtime.SetCameraInitialized(true); // Allow camera to stabilize std::this_thread::sleep_for(std::chrono::milliseconds(500)); AppendRuntimeLog("CameraManager: Camera initialized successfully"); return true; } AppendRuntimeLog("CameraManager: Failed to initialize camera"); return false; } void StartCameraSearch() { if (eyetracking_runtime.CameraSearching()) { return; } Shutdown(); AppendRuntimeLog("CameraManager: Starting camera search"); eyetracking_runtime.SetCameraSearching(true); searchThread_ = std::thread([this] { FindCameraLoop(); }); } void FindCameraLoop() { { std::lock_guard lock(camera_mutex); if (capture.isOpened() || !eyetracking_runtime.CameraSearching()) { AppendRuntimeLog("CameraManager: Search requested but camera is already open or search flag not set."); eyetracking_runtime.SetCameraSearching(false); return; } } while (eyetracking_runtime.CameraSearching()) { if (steamvrWatch.BlockFromSteamVRAbsent()) { std::this_thread::sleep_for(std::chrono::seconds(1)); continue; } try { AppendRuntimeLog("CameraManager: Attempting to find eyetracking cameras"); bool success = Initialize(); if (success) { eyetracking_runtime.SetCameraSearching(false); return; } else { AppendRuntimeLog("CameraManager: Failed to find eyetracking cameras. Trying again..."); } } catch (const std::exception& ex) { AppendRuntimeLog("CameraManager: Exception while searching for cameras: " + std::string(ex.what())); } std::this_thread::sleep_for(std::chrono::seconds(1)); } } std::string GetFrameHash(const cv::Mat& frame) { std::vector buf; cv::imencode(".jpg", frame, buf); // Simple non-cryptographic hash; sufficient for duplicate detection. std::size_t h = 0; for (unsigned char c : buf) { h = h * 131u + c; } std::ostringstream oss; oss << std::hex << h; return oss.str(); } bool GetFrame(cv::Mat& grayscale) { if (!eyetracking_runtime.CameraInitialized() || !capture.isOpened()) { return false; } cv::Mat frame; try { if (capture.grab() && capture.retrieve(frame) && !frame.empty()) { captureFailureCount_ = 0; if (frame.channels() > 1) { cv::extractChannel(frame, frame, 0); } // Check 10x10 corner for zeros if (frame.cols >= 10 && frame.rows >= 10) { cv::Mat corner = frame(cv::Rect(0, 0, 10, 10)); cv::Scalar sum = cv::sum(corner); bool allZero = (sum[0] == 0); if (frame.channels() > 1) { allZero = allZero && (sum[1] == 0) && (sum[2] == 0); if (frame.channels() > 3) { allZero = allZero && (sum[3] == 0); } } if (allZero) { AppendRuntimeLog("CameraManager: Corner 10x10 pixels sum to 0. Reinitializing cameras..."); Shutdown(); return false; } } // Duplicate frame detection duplicateDetectionTick_++; if (duplicateDetectionTick_ % 100 == 0) { duplicateDetectionTick_ = 0; std::string newHash = GetFrameHash(frame); if (newHash == lastFrameHash_) { foundDuplicateCount_++; if (foundDuplicateCount_ >= MaxDuplicateFrameDetections) { foundDuplicateCount_ = 0; Shutdown(); StartCameraSearch(); AppendRuntimeLog("Too many duplicate frames! Reinitializing cameras..."); return false; } } else { foundDuplicateCount_ = 0; } lastFrameHash_ = std::move(newHash); } grayscale = frame; width = grayscale.cols; height = grayscale.rows; return true; } else { AppendRuntimeLog("CameraManager: Failed to read frame from camera"); captureFailureCount_++; if (captureFailureCount_ >= MaxFailureAttempts) { captureFailureCount_ = 0; Shutdown(); StartCameraSearch(); AppendRuntimeLog("Could not read frames: camera disconnected. Reinitializing cameras..."); } return false; } } catch (const std::exception& ex) { AppendRuntimeLog("CameraManager: Error getting camera frame: " + std::string(ex.what())); throw; } } void Shutdown() { AppendRuntimeLog("CameraManager: Shutting down camera device"); eyetracking_runtime.SetCameraSearching(false); if (searchThread_.joinable()) { searchThread_.join(); } std::lock_guard lock(camera_mutex); if (capture.isOpened()) { AppendRuntimeLog("CameraManager: Releasing camera resource"); capture.release(); } width = 0; height = 0; eyetracking_runtime.SetCameraInitialized(false); } private: std::mutex camera_mutex; cv::VideoCapture capture; int width = 0; int height = 0; int captureFailureCount_ = 0; int duplicateDetectionTick_ = 0; int foundDuplicateCount_ = 0; std::string lastFrameHash_; std::thread searchThread_; const int MaxFailureAttempts = 3; const int MaxDuplicateFrameDetections = 5; }; bool GetFrameWithSeh(EtCamera* camera, cv::Mat* frame, bool* seh_hit) { if (seh_hit) { *seh_hit = false; } __try { return camera->GetFrame(*frame); } __except (EXCEPTION_EXECUTE_HANDLER) { if (seh_hit) { *seh_hit = true; } return false; } } class EtInference { public: void SetForceCpu(bool enabled) { force_cpu = enabled; } bool LoadModel(const std::string& model_path, std::string& error) { Shutdown(); current_model = model_path; if (model_path.empty() || !std::filesystem::exists(model_path)) { error = "Model file not found."; AppendRuntimeLog("LoadModel failed: " + error + " path=" + model_path); ready = false; return false; } if (!LoadOrt(error)) { AppendRuntimeLog("LoadOrt failed: " + error); ready = false; return false; } std::vector model_bytes = DecryptModel(model_path, error); if (!error.empty() || model_bytes.empty()) { AppendRuntimeLog("DecryptModel failed: " + error); ready = false; return false; } if (!CreateSession(model_bytes, error)) { AppendRuntimeLog("CreateSession failed: " + error); ready = false; return false; } ready = true; return true; } bool IsReady() const { return ready; } bool UsingDirectML() const { return using_directml; } MultiTaskPrediction RunEye(const cv::Mat& input_gray, bool is_right) { MultiTaskPrediction prediction{}; if (!ready || api == nullptr || session == nullptr) { return prediction; } if (input_gray.empty() || input_gray.channels() != 1) { LogOnce("RunEye aborted: invalid input (empty or not grayscale)", run_error_count); return prediction; } if (input_channels != 1) { LogOnce("RunEye aborted: unsupported input_channels=" + std::to_string(input_channels), run_error_count); return prediction; } cv::Mat resized; if (input_gray.cols != input_width || input_gray.rows != input_height) { cv::resize(input_gray, resized, cv::Size(input_width, input_height), 0, 0, cv::INTER_NEAREST); } else { resized = input_gray; } // Ensure continuous memory layout if (!resized.isContinuous()) { resized = resized.clone(); } std::vector input_tensor_data(input_width * input_height); const uint8_t* data_ptr = resized.data; for (int i = 0; i < input_width * input_height; i++) { input_tensor_data[i] = data_ptr[i] / 255.0f; } std::vector eye_side_data(1); eye_side_data[0] = is_right ? 1.0f : 0.0f; OrtValue* input_tensor = nullptr; OrtValue* eye_side_tensor = nullptr; OrtMemoryInfo* memory_info = nullptr; int64_t input_dims[4] = { 1, 1, input_height, input_width }; int64_t eye_side_dims[1] = { 1 }; OrtStatus* status = api->CreateCpuMemoryInfo(OrtArenaAllocator, OrtMemTypeDefault, &memory_info); if (status != nullptr) { api->ReleaseStatus(status); return prediction; } status = api->CreateTensorWithDataAsOrtValue( memory_info, input_tensor_data.data(), input_tensor_data.size() * sizeof(float), input_dims, 4, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &input_tensor); if (status != nullptr) { api->ReleaseStatus(status); api->ReleaseMemoryInfo(memory_info); return prediction; } status = api->CreateTensorWithDataAsOrtValue( memory_info, eye_side_data.data(), eye_side_data.size() * sizeof(float), eye_side_dims, 1, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, &eye_side_tensor); if (status != nullptr) { api->ReleaseStatus(status); api->ReleaseMemoryInfo(memory_info); api->ReleaseValue(input_tensor); return prediction; } const char* input_names[2] = { input_name.c_str(), eye_side_input_name.c_str() }; const OrtValue* input_values[2] = { input_tensor, eye_side_tensor }; const char* output_names[2] = { position_output_name.c_str(), flag_output_name.c_str() }; OrtValue* output_values[2] = { nullptr, nullptr }; size_t input_count = has_eye_side_input ? 2 : 1; size_t output_count = has_flag_output ? 2 : 1; bool seh_hit = false; status = RunOrtWithSeh(api, session, input_names, input_values, input_count, output_names, output_count, output_values, &seh_hit); if (seh_hit) { LogOnce("RunEye SEH exception during OrtApi::Run.", run_error_count); api->ReleaseMemoryInfo(memory_info); api->ReleaseValue(input_tensor); api->ReleaseValue(eye_side_tensor); return prediction; } if (status != nullptr) { std::string err = api->GetErrorMessage(status); LogOnce("RunEye OrtApi::Run failed: " + err, run_error_count); api->ReleaseStatus(status); api->ReleaseMemoryInfo(memory_info); api->ReleaseValue(input_tensor); api->ReleaseValue(eye_side_tensor); return prediction; } float* position_data = nullptr; if (output_values[0] != nullptr) { api->GetTensorMutableData(output_values[0], reinterpret_cast(&position_data)); } if (position_data != nullptr) { prediction.position[0] = position_data[0]; prediction.position[1] = position_data[1]; } float* flag_data = nullptr; if (has_flag_output && output_values[1] != nullptr) { api->GetTensorMutableData(output_values[1], reinterpret_cast(&flag_data)); prediction.flag_probability = flag_data ? flag_data[0] : 0.0f; } if (output_values[0]) { api->ReleaseValue(output_values[0]); } if (has_flag_output && output_values[1]) { api->ReleaseValue(output_values[1]); } api->ReleaseMemoryInfo(memory_info); api->ReleaseValue(input_tensor); api->ReleaseValue(eye_side_tensor); return prediction; } private: bool LoadOrt(std::string& error) { std::string dll_path = "dependencies\\onnxruntime\\bin\\onnxruntime.dll"; onnxruntime_dll = LoadLibraryA(dll_path.c_str()); if (!onnxruntime_dll) { onnxruntime_dll = LoadLibraryA("onnxruntime.dll"); } if (!onnxruntime_dll) { error = "onnxruntime.dll not found."; AppendRuntimeLog("LoadOrt failed: " + error); return false; } auto get_api_base = reinterpret_cast(GetProcAddress(onnxruntime_dll, "OrtGetApiBase")); if (!get_api_base) { error = "OrtGetApiBase not found."; AppendRuntimeLog("LoadOrt failed: " + error); return false; } const OrtApiBase* base = get_api_base(); if (base) { for (int v = ORT_API_VERSION; v >= 1; --v) { api = base->GetApi(static_cast(v)); if (api) { std::ostringstream msg; msg << "OrtApi resolved with version " << v; AppendRuntimeLog(msg.str()); break; } } } if (!api) { std::string version = base ? base->GetVersionString() : "unknown"; error = "Failed to get OrtApi (runtime version " + version + ")."; AppendRuntimeLog("LoadOrt failed: " + error); return false; } OrtStatus* status = api->CreateEnv(ORT_LOGGING_LEVEL_WARNING, "BeyondEyetracking", &env); if (status != nullptr) { error = api->GetErrorMessage(status); api->ReleaseStatus(status); AppendRuntimeLog("LoadOrt failed: " + error); return false; } status = api->GetAllocatorWithDefaultOptions(&allocator); if (status != nullptr) { error = api->GetErrorMessage(status); api->ReleaseStatus(status); AppendRuntimeLog("LoadOrt failed: " + error); return false; } return true; } std::vector DecryptModel(const std::string& path, std::string& error) { std::ifstream in(path, std::ios::binary); if (!in.is_open()) { error = "Failed to open model file."; AppendRuntimeLog("DecryptModel failed: " + error + " path=" + path); return {}; } std::vector data((std::istreambuf_iterator(in)), std::istreambuf_iterator()); if (data.empty()) { error = "Model file empty."; AppendRuntimeLog("DecryptModel failed: " + error + " path=" + path); return {}; } const std::string part1 = "gFlFTSp-sW-"; const std::string part2 = "HI"; const std::string part3 = "wuQvfbLcI1u6wsRc2b"; const std::string part4 = "3-eOb5SpPc_Y="; std::string key = part1 + part2 + part3 + part4; std::vector result(data.size()); for (size_t i = 0; i < data.size(); i++) { result[i] = static_cast(data[i] ^ key[i % key.size()]); } return result; } bool CreateSession(const std::vector& model_bytes, std::string& error) { auto reset_session_options = [&]() { if (session_options) { api->ReleaseSessionOptions(session_options); session_options = nullptr; } }; auto try_create = [&](bool enable_dml, bool* dml_seh, bool* session_seh) -> bool { OrtStatus* status = api->CreateSessionOptions(&session_options); if (status != nullptr) { error = api->GetErrorMessage(status); api->ReleaseStatus(status); AppendRuntimeLog("CreateSession failed: " + error); return false; } // Allow models with unreleased opsets (e.g. 17) so our models work in Release builds status = api->AddSessionConfigEntry(session_options, "session.allow_released_opsets_only", "0"); if (status != nullptr) { api->ReleaseStatus(status); } using_directml = false; if (enable_dml) { using_dml_fn = reinterpret_cast( GetProcAddress(onnxruntime_dll, "OrtSessionOptionsAppendExecutionProvider_DML")); if (using_dml_fn) { bool seh_hit = false; OrtStatus* dml_status = nullptr; if (!TryAppendDmlWithSeh(using_dml_fn, session_options, &dml_status, &seh_hit) || seh_hit) { AppendRuntimeLog("SEH during DirectML provider append."); if (dml_seh) { *dml_seh = true; } reset_session_options(); return false; } if (dml_status == nullptr) { using_directml = true; AppendRuntimeLog("DirectML enabled."); } else { std::string dml_error = api->GetErrorMessage(dml_status); AppendRuntimeLog("DirectML init failed: " + dml_error); api->ReleaseStatus(dml_status); } } else { DWORD proc_err = GetLastError(); AppendRuntimeLog("DirectML provider not found in onnxruntime.dll; using CPU. (GetProcAddress err=" + std::to_string(proc_err) + ")"); } } else { AppendRuntimeLog("DirectML disabled (force CPU)."); } api->SetIntraOpNumThreads(session_options, 1); api->SetInterOpNumThreads(session_options, 1); bool seh_hit = false; OrtStatus* create_status = nullptr; if (!TryCreateSessionFromArrayWithSeh(api, env, model_bytes.data(), model_bytes.size(), session_options, &session, &create_status, &seh_hit) || seh_hit) { if (session != nullptr) { api->ReleaseSession(session); session = nullptr; } AppendRuntimeLog("SEH during CreateSessionFromArray."); if (session_seh) { *session_seh = true; } reset_session_options(); return false; } if (create_status != nullptr) { error = api->GetErrorMessage(create_status); api->ReleaseStatus(create_status); AppendRuntimeLog("CreateSession failed: " + error); reset_session_options(); return false; } return true; }; bool dml_seh = false; bool session_seh = false; bool want_dml = !force_cpu; if (!try_create(want_dml, &dml_seh, &session_seh)) { if (want_dml) { AppendRuntimeLog("Retrying session creation without DirectML."); using_directml = false; dml_seh = false; session_seh = false; if (!try_create(false, &dml_seh, &session_seh)) { return false; } AppendRuntimeLog("CPU fallback session creation succeeded."); } else { return false; } } size_t input_count = 0; size_t output_count = 0; api->SessionGetInputCount(session, &input_count); api->SessionGetOutputCount(session, &output_count); AppendRuntimeLog("CreateSession inputs=" + std::to_string(input_count) + " outputs=" + std::to_string(output_count)); if (input_count == 0 || output_count == 0) { error = "Model has no inputs/outputs."; AppendRuntimeLog("CreateSession failed: " + error); return false; } char* in_name = nullptr; api->SessionGetInputName(session, 0, allocator, &in_name); input_name = in_name ? in_name : ""; if (in_name) { api->AllocatorFree(allocator, in_name); } if (input_count > 1) { char* eye_name = nullptr; api->SessionGetInputName(session, 1, allocator, &eye_name); eye_side_input_name = eye_name ? eye_name : ""; if (eye_name) { api->AllocatorFree(allocator, eye_name); } has_eye_side_input = !eye_side_input_name.empty(); } else { eye_side_input_name = ""; has_eye_side_input = false; } char* out_name = nullptr; api->SessionGetOutputName(session, 0, allocator, &out_name); position_output_name = out_name ? out_name : "position"; if (out_name) { api->AllocatorFree(allocator, out_name); } if (output_count > 1) { char* flag_name = nullptr; api->SessionGetOutputName(session, 1, allocator, &flag_name); flag_output_name = flag_name ? flag_name : "flag_prob"; if (flag_name) { api->AllocatorFree(allocator, flag_name); } has_flag_output = true; } else { flag_output_name = "flag_prob"; has_flag_output = false; } OrtTypeInfo* type_info = nullptr; api->SessionGetInputTypeInfo(session, 0, &type_info); if (type_info) { const OrtTensorTypeAndShapeInfo* tensor_info = nullptr; api->CastTypeInfoToTensorInfo(type_info, &tensor_info); size_t dim_count = 0; api->GetDimensionsCount(tensor_info, &dim_count); if (dim_count >= 4) { int64_t dims[4]{}; api->GetDimensions(tensor_info, dims, 4); input_channels = static_cast(dims[1]); input_height = static_cast(dims[2]); input_width = static_cast(dims[3]); if (input_height <= 0) input_height = 256; if (input_width <= 0) input_width = 256; if (input_channels <= 0) input_channels = 1; AppendRuntimeLog("Input shape NCHW: 1x" + std::to_string(input_channels) + "x" + std::to_string(input_height) + "x" + std::to_string(input_width)); } api->ReleaseTypeInfo(type_info); } if (input_channels <= 0) { input_channels = 1; } return true; } void Shutdown() { if (session) { api->ReleaseSession(session); session = nullptr; } if (session_options) { api->ReleaseSessionOptions(session_options); session_options = nullptr; } if (env) { api->ReleaseEnv(env); env = nullptr; } if (onnxruntime_dll) { FreeLibrary(onnxruntime_dll); onnxruntime_dll = nullptr; } api = nullptr; allocator = nullptr; using_directml = true; ready = false; } std::string current_model; bool ready = false; bool using_directml = true; HMODULE onnxruntime_dll = nullptr; const OrtApi* api = nullptr; OrtEnv* env = nullptr; OrtSession* session = nullptr; OrtSessionOptions* session_options = nullptr; OrtAllocator* allocator = nullptr; OrtStatus* (ORT_API_CALL* using_dml_fn)(OrtSessionOptions*, int) = nullptr; std::string input_name; std::string eye_side_input_name; std::string position_output_name = "position"; std::string flag_output_name = "flag_prob"; int input_height = 256; int input_width = 256; int input_channels = 1; bool has_eye_side_input = true; bool has_flag_output = true; int run_error_count = 0; bool force_cpu = false; }; struct SharedGazeData { float LeftEyeX; float LeftEyeY; float LeftEyeZ; float RightEyeX; float RightEyeY; float RightEyeZ; float CombinedX; float CombinedY; float CombinedZ; float Confidence; int64_t Timestamp; int32_t IsValid; int32_t LeftEyeFlagCondition; int32_t RightEyeFlagCondition; }; struct VrcftSharedGazeData { float LeftEyeX; float LeftEyeY; float LeftEyeZ; float RightEyeX; float RightEyeY; float RightEyeZ; float CombinedX; float CombinedY; float CombinedZ; float Confidence; int64_t Timestamp; int32_t IsValid; float LeftEyeClosedAmount; float RightEyeClosedAmount; }; class MemmapPublisher { public: bool Initialize(std::string& error) { Close(); mapping = CreateFileMappingA(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, 0, sizeof(SharedGazeData), "EyeTrackingMemmapData"); if (!mapping) { error = "Failed to create memmap."; return false; } view = MapViewOfFile(mapping, FILE_MAP_ALL_ACCESS, 0, 0, sizeof(SharedGazeData)); if (!view) { CloseHandle(mapping); mapping = nullptr; error = "Failed to map view."; return false; } SharedGazeData zero{}; zero.IsValid = 0; std::memcpy(view, &zero, sizeof(SharedGazeData)); return true; } void Close() { if (view) { SharedGazeData zero{}; zero.IsValid = 0; std::memcpy(view, &zero, sizeof(SharedGazeData)); UnmapViewOfFile(view); view = nullptr; } if (mapping) { CloseHandle(mapping); mapping = nullptr; } } void Process(const EyeTrackingData& data, const FrustumLoader& frustum) { if (!view || !frustum.IsLoaded()) { return; } float left_x = data.left_x; float left_y = data.left_y; float right_x = data.right_x; float right_y = data.right_y; bool single = (right_x == 0.0f && right_y == 0.0f); if (single) { right_x = left_x; right_y = left_y; } Vec2 left_angles = frustum.ScreenPositionToViewAngles(Vec2{ left_x, left_y }, true); Vec2 right_angles = frustum.ScreenPositionToViewAngles(Vec2{ right_x, right_y }, false); Vec3 left_vec = AnglesToVector(left_angles.y, left_angles.x); Vec3 right_vec = AnglesToVector(right_angles.y, right_angles.x); Vec3 combined{ (left_vec.x + right_vec.x) * 0.5f, (left_vec.y + right_vec.y) * 0.5f, (left_vec.z + right_vec.z) * 0.5f }; float len = std::sqrt(combined.x * combined.x + combined.y * combined.y + combined.z * combined.z); if (len > 0.0f) { combined.x /= len; combined.y /= len; combined.z /= len; } SharedGazeData out{}; out.LeftEyeX = left_vec.x; out.LeftEyeY = left_vec.y; out.LeftEyeZ = left_vec.z; out.RightEyeX = right_vec.x; out.RightEyeY = right_vec.y; out.RightEyeZ = right_vec.z; out.CombinedX = combined.x; out.CombinedY = combined.y; out.CombinedZ = combined.z; out.Confidence = data.confidence; out.Timestamp = data.timestamp_ms; out.IsValid = 1; out.LeftEyeFlagCondition = data.left_flag_condition ? 1 : 0; out.RightEyeFlagCondition = data.right_flag_condition ? 1 : 0; std::memcpy(view, &out, sizeof(SharedGazeData)); } private: Vec3 AnglesToVector(float pitch, float yaw) { float pitch_rad = pitch * 3.14159265f / 180.0f; float yaw_rad = yaw * 3.14159265f / 180.0f; float x = std::sin(yaw_rad) * std::cos(pitch_rad); float y = std::sin(pitch_rad); float z = std::cos(yaw_rad) * std::cos(pitch_rad); return Vec3{ x, y, -z }; } HANDLE mapping = nullptr; void* view = nullptr; }; class VRCFTPublisher { public: VRCFTPublisher() { InitializeFilters(); buffer = std::make_unique( 40, 10, [this](const std::vector& chunk) { return AnalyzeChunk(chunk); }, [this](const EyeTrackingData& data) { ProcessData(data); }, [this]() { return running && view != nullptr; }); } bool Initialize(std::string& error) { Close(); AppendRuntimeLog("VRCFT init: struct_size=" + std::to_string(sizeof(VrcftSharedGazeData))); mapping = CreateFileMappingA(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, 0, sizeof(VrcftSharedGazeData), "VRCFTMemmapData"); if (!mapping) { error = "Failed to create VRCFT memmap."; return false; } view = MapViewOfFile(mapping, FILE_MAP_ALL_ACCESS, 0, 0, sizeof(VrcftSharedGazeData)); if (!view) { CloseHandle(mapping); mapping = nullptr; error = "Failed to map VRCFT view."; return false; } SECURITY_DESCRIPTOR sd{}; SECURITY_ATTRIBUTES sa{}; bool sd_ok = InitializeSecurityDescriptor(&sd, SECURITY_DESCRIPTOR_REVISION) != 0; bool dacl_ok = sd_ok && SetSecurityDescriptorDacl(&sd, TRUE, nullptr, FALSE) != 0; if (dacl_ok) { sa.nLength = sizeof(sa); sa.lpSecurityDescriptor = &sd; sa.bInheritHandle = FALSE; mapping_global = CreateFileMappingA(INVALID_HANDLE_VALUE, &sa, PAGE_READWRITE, 0, sizeof(VrcftSharedGazeData), "Global\\VRCFTMemmapData"); } else { AppendRuntimeLog("VRCFT global memmap security init failed (err=" + std::to_string(GetLastError()) + ")."); } if (mapping_global) { view_global = MapViewOfFile(mapping_global, FILE_MAP_ALL_ACCESS, 0, 0, sizeof(VrcftSharedGazeData)); if (!view_global) { CloseHandle(mapping_global); mapping_global = nullptr; } else { AppendRuntimeLog("VRCFT global memmap active."); } } else { AppendRuntimeLog("VRCFT global memmap not created (err=" + std::to_string(GetLastError()) + ")."); } VrcftSharedGazeData zero{}; zero.IsValid = 0; std::memcpy(view, &zero, sizeof(VrcftSharedGazeData)); if (view_global) { std::memcpy(view_global, &zero, sizeof(VrcftSharedGazeData)); } running = true; return true; } void Close() { running = false; if (buffer) { buffer->Clear(); } blink_lerp.Reset(); if (view) { VrcftSharedGazeData zero{}; zero.IsValid = 0; std::memcpy(view, &zero, sizeof(VrcftSharedGazeData)); UnmapViewOfFile(view); view = nullptr; } if (view_global) { VrcftSharedGazeData zero{}; zero.IsValid = 0; std::memcpy(view_global, &zero, sizeof(VrcftSharedGazeData)); UnmapViewOfFile(view_global); view_global = nullptr; } if (mapping) { CloseHandle(mapping); mapping = nullptr; } if (mapping_global) { CloseHandle(mapping_global); mapping_global = nullptr; } } void Configure(float smoothing, bool smoothing_enabled, EtBlinkMode blink_mode) { beta = smoothing; filter_enabled = smoothing_enabled; eye_closure_mode = blink_mode; if (eye_closure_mode == EtBlinkMode::None) { post_blink_hold = 0; blink_lerp.Reset(); } InitializeFilters(); } void UpdateFrustum(const FrustumLoader& loader) { frustum = loader; } void OnData(const EyeTrackingData& data) { if (buffer) { buffer->AddData(data); } } private: void InitializeFilters() { left_x_filter = OneEuroFilter(1.0f, beta, 1.0f); left_y_filter = OneEuroFilter(1.0f, beta, 1.0f); right_x_filter = OneEuroFilter(1.0f, beta, 1.0f); right_y_filter = OneEuroFilter(1.0f, beta, 1.0f); } EyeTrackingData AnalyzeChunk(const std::vector& chunk) { EyeTrackingData frame = chunk.empty() ? EyeTrackingData{} : chunk.back(); if (eye_closure_mode == EtBlinkMode::None) { post_blink_hold = 0; return frame; } if (frame.HasAnyFlagCondition()) { post_blink_hold = blink_smoothing_frames; return frame; } if (post_blink_hold > 0) { post_blink_hold--; return last_stable_frame; } last_stable_frame = frame; return frame; } void ProcessData(const EyeTrackingData& data) { if (!running || !view || !frustum.IsLoaded()) { if (running && view && !frustum.IsLoaded()) { LogOnce("VRCFT skipped: frustum not loaded.", write_log_count); } return; } bool filter_disabled = !filter_enabled; double current_time = GetTicksMsec() / 1000.0; bool left_blink = data.left_flag_condition; bool right_blink = data.right_flag_condition; float left_base_closed = 1.0f - std::clamp(data.left_openness, 0.0f, 1.0f); float right_base_closed = 1.0f - std::clamp(data.right_openness, 0.0f, 1.0f); float left_closed = 0.0f; float right_closed = 0.0f; bool treat_as_blinking_or_lerping = false; switch (eye_closure_mode) { case EtBlinkMode::Blinking: { bool is_blinking = left_blink || right_blink; float avg_base_closed = (left_base_closed + right_base_closed) * 0.5f; float both_closed = blink_lerp.UpdateBlinkLerp(is_blinking, current_time, avg_base_closed); left_closed = both_closed; right_closed = both_closed; treat_as_blinking_or_lerping = is_blinking || blink_lerp.IsLerping(); break; } case EtBlinkMode::Winking: { auto per_eye_closed = blink_lerp.UpdateBlinkLerpPerEye( left_blink, right_blink, current_time, left_base_closed, right_base_closed); left_closed = per_eye_closed.first; right_closed = per_eye_closed.second; bool left_wink = data.left_flag_condition && !data.right_flag_condition; bool right_wink = !data.left_flag_condition && data.right_flag_condition; if (left_wink) { left_closed = 1.0f; right_closed = 0.0f; } else if (right_wink) { right_closed = 1.0f; left_closed = 0.0f; } treat_as_blinking_or_lerping = left_blink || right_blink || blink_lerp.IsLerping(); break; } case EtBlinkMode::None: default: blink_lerp.Reset(); left_closed = 0.0f; right_closed = 0.0f; treat_as_blinking_or_lerping = false; break; } float left_x = data.left_x; float left_y = data.left_y; float right_x = data.right_x; float right_y = data.right_y; float filtered_left_x = left_x_filter.Filter(data.left_x, current_time); float filtered_left_y = left_y_filter.Filter(data.left_y, current_time); float filtered_right_x = right_x_filter.Filter(data.right_x, current_time); float filtered_right_y = right_y_filter.Filter(data.right_y, current_time); if (!treat_as_blinking_or_lerping && !filter_disabled) { left_x = filtered_left_x; left_y = filtered_left_y; right_x = filtered_right_x; right_y = filtered_right_y; } bool single = (data.right_x == 0.0f && data.right_y == 0.0f); if (single) { right_x = left_x; right_y = left_y; } Vec2 left_angles = frustum.ScreenPositionToViewAngles(Vec2{ left_x, left_y }, true); Vec2 right_angles = frustum.ScreenPositionToViewAngles(Vec2{ right_x, right_y }, false); auto [conv_left, conv_right] = EyeConvergence::Apply(Vec2{ left_angles.x, left_angles.y }, Vec2{ right_angles.x, right_angles.y }); Vec3 left_vec = AnglesToVector(conv_left.y, conv_left.x); Vec3 right_vec = AnglesToVector(conv_right.y, conv_right.x); Vec3 combined{ (left_vec.x + right_vec.x) * 0.5f, (left_vec.y + right_vec.y) * 0.5f, (left_vec.z + right_vec.z) * 0.5f }; float len = std::sqrt(combined.x * combined.x + combined.y * combined.y + combined.z * combined.z); if (len > 0.0f) { combined.x /= len; combined.y /= len; combined.z /= len; } VrcftSharedGazeData out{}; out.LeftEyeX = left_vec.x; out.LeftEyeY = left_vec.y; out.LeftEyeZ = left_vec.z; out.RightEyeX = right_vec.x; out.RightEyeY = right_vec.y; out.RightEyeZ = right_vec.z; out.CombinedX = combined.x; out.CombinedY = combined.y; out.CombinedZ = combined.z; out.Confidence = data.confidence; out.Timestamp = data.timestamp_ms; out.IsValid = 1; out.LeftEyeClosedAmount = left_closed; out.RightEyeClosedAmount = right_closed; double now_ms = GetTicksMsec(); std::memcpy(view, &out, sizeof(VrcftSharedGazeData)); if (view_global) { std::memcpy(view_global, &out, sizeof(VrcftSharedGazeData)); } if (now_ms - last_write_log_ms > 10000) { last_write_log_ms = now_ms; AppendRuntimeLog("VRCFT wrote sample ts=" + std::to_string(out.Timestamp)); } } Vec3 AnglesToVector(float pitch, float yaw) { float pitch_rad = pitch * 3.14159265f / 180.0f; float yaw_rad = yaw * 3.14159265f / 180.0f; float x = std::sin(yaw_rad) * std::cos(pitch_rad); float y = std::sin(pitch_rad); float z = std::cos(yaw_rad) * std::cos(pitch_rad); return Vec3{ x, y, -z }; } HANDLE mapping = nullptr; void* view = nullptr; HANDLE mapping_global = nullptr; void* view_global = nullptr; bool running = false; float beta = 0.8f; bool filter_enabled = true; int blink_smoothing_frames = 3; int post_blink_hold = 0; EyeTrackingData last_stable_frame{}; BlinkLerp blink_lerp{}; OneEuroFilter left_x_filter; OneEuroFilter left_y_filter; OneEuroFilter right_x_filter; OneEuroFilter right_y_filter; FrustumLoader frustum{}; std::unique_ptr buffer; uint64_t last_write_log_ms = 0; int write_log_count = 0; EtBlinkMode eye_closure_mode = EtBlinkMode::Blinking; }; class VRChatPublisher { public: VRChatPublisher() { InitializeFilters(); buffer = std::make_unique( 40, 10, [this](const std::vector& chunk) { return AnalyzeChunk(chunk); }, [this](const EyeTrackingData& data) { ProcessData(data); }, []() { return true; }); } bool Initialize(const std::string& ip, int port, std::string& error) { return osc.Initialize(ip, port, error); } void Configure(float smoothing, bool smoothing_enabled, EtBlinkMode blink_mode) { beta = smoothing; filter_enabled = smoothing_enabled; eye_closure_mode = blink_mode; if (eye_closure_mode == EtBlinkMode::None) { post_blink_hold = 0; blink_lerp.Reset(); } InitializeFilters(); } void UpdateFrustum(const FrustumLoader& loader) { frustum = loader; } void OnData(const EyeTrackingData& data) { if (buffer) { buffer->AddData(data); } } void Shutdown() { osc.Shutdown(); } private: void InitializeFilters() { left_x_filter = OneEuroFilter(1.0f, beta, 1.0f); left_y_filter = OneEuroFilter(1.0f, beta, 1.0f); right_x_filter = OneEuroFilter(1.0f, beta, 1.0f); right_y_filter = OneEuroFilter(1.0f, beta, 1.0f); } EyeTrackingData AnalyzeChunk(const std::vector& chunk) { EyeTrackingData frame = chunk.empty() ? EyeTrackingData{} : chunk.back(); if (eye_closure_mode == EtBlinkMode::None) { post_blink_hold = 0; return frame; } if (frame.HasAnyFlagCondition()) { post_blink_hold = blink_smoothing_frames; return frame; } if (post_blink_hold > 0) { post_blink_hold--; return last_stable_frame; } last_stable_frame = frame; return frame; } void ProcessData(const EyeTrackingData& data) { double current_time = GetTicksMsec() / 1000.0; bool filter_disabled = !filter_enabled; bool is_blinking = eye_closure_mode != EtBlinkMode::None && data.HasAnyFlagCondition(); float left_open = std::clamp(data.left_openness, 0.0f, 1.0f); float right_open = std::clamp(data.right_openness, 0.0f, 1.0f); float avg_open = (left_open + right_open) * 0.5f; float base_closed = std::clamp(1.0f - avg_open, 0.0f, 1.0f); float eyes_closed = 0.0f; if (eye_closure_mode == EtBlinkMode::None) { blink_lerp.Reset(); } else { eyes_closed = blink_lerp.UpdateBlinkLerp(is_blinking, current_time, base_closed); } float left_x = filter_disabled ? data.left_x : left_x_filter.Filter(data.left_x, current_time); float left_y = filter_disabled ? data.left_y : left_y_filter.Filter(data.left_y, current_time); float right_x = data.right_x; float right_y = data.right_y; bool single = (right_x == 0.0f && right_y == 0.0f); if (single) { right_x = left_x; right_y = left_y; } else if (!filter_disabled) { right_x = right_x_filter.Filter(right_x, current_time); right_y = right_y_filter.Filter(right_y, current_time); } float left_pitch = 0.0f, left_yaw = 0.0f, right_pitch = 0.0f, right_yaw = 0.0f; if (frustum.IsLoaded()) { auto [conv_left, conv_right] = EyeConvergence::Apply(Vec2{ left_x, left_y }, Vec2{ right_x, right_y }); Vec2 left_angles = frustum.ScreenPositionToViewAngles(conv_left, true); Vec2 right_angles = frustum.ScreenPositionToViewAngles(conv_right, false); left_yaw = left_angles.x; right_yaw = right_angles.x; left_pitch = -left_angles.y; right_pitch = -right_angles.y; } osc.SendMessage("/tracking/eye/LeftRightPitchYaw", std::array{ left_pitch, left_yaw, right_pitch, right_yaw }.data(), 4); osc.SendMessage("/tracking/eye/EyesClosedAmount", eyes_closed); } OscClient osc{}; float beta = 0.8f; bool filter_enabled = true; int blink_smoothing_frames = 3; int post_blink_hold = 0; EyeTrackingData last_stable_frame{}; BlinkLerp blink_lerp{}; OneEuroFilter left_x_filter; OneEuroFilter left_y_filter; OneEuroFilter right_x_filter; OneEuroFilter right_y_filter; FrustumLoader frustum{}; std::unique_ptr buffer; EtBlinkMode eye_closure_mode = EtBlinkMode::Blinking; }; class ViewerPublisher { public: ViewerPublisher() { InitializeFilters(); buffer = std::make_unique( 40, 10, [this](const std::vector& chunk) { return AnalyzeChunk(chunk); }, [this](const EyeTrackingData& data) { ProcessData(data); }, []() { return true; }); } void Configure(float smoothing, bool smoothing_enabled) { beta = smoothing; filter_enabled = smoothing_enabled; InitializeFilters(); } void Close() { if (cv::getWindowProperty("Left Eye", cv::WindowPropertyFlags::WND_PROP_VISIBLE) >= 1) { cv::destroyWindow("Left Eye"); } if (cv::getWindowProperty("Right Eye", cv::WindowPropertyFlags::WND_PROP_VISIBLE) >= 1) { cv::destroyWindow("Right Eye"); } } void OnData(const EyeTrackingData& data) { if (buffer) { buffer->AddData(data); } } private: void InitializeFilters() { left_x_filter = OneEuroFilter(1.0f, beta, 1.0f); left_y_filter = OneEuroFilter(1.0f, beta, 1.0f); right_x_filter = OneEuroFilter(1.0f, beta, 1.0f); right_y_filter = OneEuroFilter(1.0f, beta, 1.0f); } EyeTrackingData AnalyzeChunk(const std::vector& chunk) { EyeTrackingData frame = chunk.empty() ? EyeTrackingData{} : chunk.back(); return frame; } void ProcessData(const EyeTrackingData& data) { double current_time = GetTicksMsec() / 1000.0; bool filter_disabled = !filter_enabled; float left_x = filter_disabled ? data.left_x : left_x_filter.Filter(data.left_x, current_time); float left_y = filter_disabled ? data.left_y : left_y_filter.Filter(data.left_y, current_time); float right_x = filter_disabled ? data.right_x : right_x_filter.Filter(data.right_x, current_time); float right_y = filter_disabled ? data.right_y : right_y_filter.Filter(data.right_y, current_time); float window_scale = g_Dpi.scale; if (!data.left_eye_image.empty()) { cv::Mat leftEyeWithOverlay = cv::Mat(); cv::flip(data.left_eye_image, leftEyeWithOverlay, 1); if (window_scale != 1) { cv::resize(leftEyeWithOverlay, leftEyeWithOverlay, cv::Size(), window_scale, window_scale, cv::INTER_LINEAR); } DrawGrid(leftEyeWithOverlay); // Draw raw eye position (white) DrawEyePosition(leftEyeWithOverlay, data.left_x, data.left_y, cv::Scalar(100, 100, 100), "Raw", 0, 0); // Draw filtered eye position (green) DrawEyePosition(leftEyeWithOverlay, left_x, left_y, cv::Scalar(255, 255, 255), "Filtered", 0, -15); cv::imshow("Left Eye", leftEyeWithOverlay); } if (!data.right_eye_image.empty()) { cv::Mat rightEyeWithOverlay = cv::Mat(); cv::flip(data.right_eye_image, rightEyeWithOverlay, 1); if (window_scale != 1) { cv::resize(rightEyeWithOverlay, rightEyeWithOverlay, cv::Size(), window_scale, window_scale, cv::INTER_LINEAR); } DrawGrid(rightEyeWithOverlay); // Draw raw eye position (white) DrawEyePosition(rightEyeWithOverlay, data.right_x, data.right_y, cv::Scalar(100, 100, 100), "Raw", 0, 0); // Draw filtered eye position (green) DrawEyePosition(rightEyeWithOverlay, right_x, right_y, cv::Scalar(255, 255, 255), "Filtered", 0, -15); cv::imshow("Right Eye", rightEyeWithOverlay); } cv::waitKey(1); } void DrawGrid(cv::Mat image) { int height = image.rows; int width = image.cols; // Draw vertical lines for (int i = 0; i < 5; i++) { int x = (int)(width * i / 4.0); cv::line(image, cv::Point(x, 0), cv::Point(x, height), cv::Scalar(128, 128, 128), 1); // Draw coordinate float coord = (float)i / 4.0f; cv::String label = std::format("{:.1f}", coord); cv::putText( image, label, cv::Point(x + 5, 20), cv::FONT_HERSHEY_SIMPLEX, 0.5 * g_Dpi.scale, cv::Scalar(255, 255, 255), 1 ); } // Draw horizontal lines for (int i = 0; i < 5; i++) { int y = (int)(height * i / 4.0); cv::line(image, cv::Point(0, y), cv::Point(width, y), cv::Scalar(128, 128, 128), 1); // Draw coordinate float coord = (float)i / 4.0f; cv::String label = std::format("{:.1f}", coord); cv::putText( image, label, cv::Point(5, y + 20), cv::FONT_HERSHEY_SIMPLEX, 0.5 * g_Dpi.scale, cv::Scalar(255, 255, 255), 1 ); } } void DrawEyePosition(cv::Mat image, float eyeX, float eyeY, cv::Scalar color, cv::String label, float labelOffsetX, float labelOffsetY) { int height = image.rows; int width = image.cols; // Convert normalized coordinates to pixel coordinates int pixelX = (int)(eyeX * width); int pixelY = (int)((1 - eyeY) * height); // Flip Y coordinate (0 at top in image) cv::String p_label = std::format("{}: ({:.2f}, {:.2f})", label, eyeX, eyeY); // Draw circle at eye position cv::circle(image, cv::Point(pixelX, pixelY), int(5 * g_Dpi.scale), color, -1); // Draw coordinate values with label cv::putText( image, p_label, cv::Point(pixelX + int((10 + labelOffsetX) * g_Dpi.scale), pixelY + int(labelOffsetY * g_Dpi.scale)), cv::FONT_HERSHEY_SIMPLEX, 0.5 * g_Dpi.scale, color, 1 ); } float beta = 0.8f; bool filter_enabled = true; OneEuroFilter left_x_filter; OneEuroFilter left_y_filter; OneEuroFilter right_x_filter; OneEuroFilter right_y_filter; std::unique_ptr buffer; }; struct EtRuntimeConfig { bool enabled = false; bool force_cpu = false; bool publish_memmap = true; bool publish_vrcft = false; bool publish_vrchat = false; bool publish_viewer = false; bool smoothing_enabled = true; float smoothing_intensity = 0.8f; bool left_eye_enabled = true; bool right_eye_enabled = true; bool enabled_alignment_helper = false; float flag_threshold = 0.5f; EtBlinkMode blink_mode = EtBlinkMode::Blinking; std::string model_path; std::string osc_ip = "127.0.0.1"; int osc_port = 9000; }; bool RunEyeWithSeh(EtInference* inference, const cv::Mat* eye, bool is_right, MultiTaskPrediction* out, bool* seh_hit) { if (seh_hit) { *seh_hit = false; } __try { if (out) { *out = inference->RunEye(*eye, is_right); } return true; } __except (EXCEPTION_EXECUTE_HANDLER) { if (seh_hit) { *seh_hit = true; } return false; } } bool MemmapProcessWithSeh(MemmapPublisher* memmap, const EyeTrackingData* data, const FrustumLoader* frustum, bool* seh_hit) { if (seh_hit) { *seh_hit = false; } __try { memmap->Process(*data, *frustum); return true; } __except (EXCEPTION_EXECUTE_HANDLER) { if (seh_hit) { *seh_hit = true; } return false; } } bool VrcftOnDataWithSeh(VRCFTPublisher* vrcft, const EyeTrackingData* data, bool* seh_hit) { if (seh_hit) { *seh_hit = false; } __try { vrcft->OnData(*data); return true; } __except (EXCEPTION_EXECUTE_HANDLER) { if (seh_hit) { *seh_hit = true; } return false; } } bool VrchatOnDataWithSeh(VRChatPublisher* vrchat, const EyeTrackingData* data, bool* seh_hit) { if (seh_hit) { *seh_hit = false; } __try { vrchat->OnData(*data); return true; } __except (EXCEPTION_EXECUTE_HANDLER) { if (seh_hit) { *seh_hit = true; } return false; } } bool ViewerOnDataWithSeh(ViewerPublisher* viewer, const EyeTrackingData* data, bool* seh_hit) { if (seh_hit) { *seh_hit = false; } __try { viewer->OnData(*data); return true; } __except (EXCEPTION_EXECUTE_HANDLER) { if (seh_hit) { *seh_hit = true; } return false; } } } EyeTrackingRuntime eyetracking_runtime; static EtRuntimeConfig g_config; static std::mutex g_config_mutex; static std::atomic g_config_dirty{ false }; static std::atomic g_config_log_count{ 0 }; void EyeTrackingRuntime::Configure(const UI_Run_State& rs) { std::lock_guard guard(g_config_mutex); EtRuntimeConfig next{}; next.enabled = rs.et_enabled; next.publish_memmap = rs.et_publish_memmap; next.publish_vrcft = rs.et_publish_vrcft; next.publish_vrchat = rs.et_publish_vrchat; next.publish_viewer = rs.et_publish_viewer; next.smoothing_enabled = rs.et_smoothing_enabled; next.smoothing_intensity = rs.et_smoothing_intensity; next.left_eye_enabled = rs.et_left_eye_enabled; next.right_eye_enabled = rs.et_right_eye_enabled; next.flag_threshold = rs.et_flag_threshold; next.blink_mode = NormalizeBlinkMode(rs.et_blink_mode); next.model_path = rs.et_model_path; next.osc_ip = rs.et_osc_ip; next.osc_port = rs.et_osc_port; next.enabled_alignment_helper = rs.enabled_alignment_helper; bool changed = g_config.enabled != next.enabled || g_config.publish_memmap != next.publish_memmap || g_config.publish_vrcft != next.publish_vrcft || g_config.publish_vrchat != next.publish_vrchat || g_config.publish_viewer != next.publish_viewer || g_config.smoothing_enabled != next.smoothing_enabled || std::fabs(g_config.smoothing_intensity - next.smoothing_intensity) > 0.0001f || g_config.left_eye_enabled != next.left_eye_enabled || g_config.right_eye_enabled != next.right_eye_enabled || std::fabs(g_config.flag_threshold - next.flag_threshold) > 0.0001f || g_config.blink_mode != next.blink_mode || g_config.model_path != next.model_path || g_config.osc_ip != next.osc_ip || g_config.osc_port != next.osc_port || g_config.enabled_alignment_helper != next.enabled_alignment_helper; if (changed) { if (g_config_log_count.load() < 8) { std::vector diffs; if (g_config.enabled != next.enabled) diffs.push_back("enabled"); if (g_config.publish_memmap != next.publish_memmap) diffs.push_back("memmap"); if (g_config.publish_vrcft != next.publish_vrcft) diffs.push_back("vrcft"); if (g_config.publish_vrchat != next.publish_vrchat) diffs.push_back("vrchat"); if (g_config.publish_viewer != next.publish_viewer) diffs.push_back("viewer"); if (g_config.smoothing_enabled != next.smoothing_enabled) diffs.push_back("smoothing_enabled"); if (std::fabs(g_config.smoothing_intensity - next.smoothing_intensity) > 0.0001f) diffs.push_back("smoothing_intensity"); if (g_config.left_eye_enabled != next.left_eye_enabled) diffs.push_back("left_eye"); if (g_config.right_eye_enabled != next.right_eye_enabled) diffs.push_back("right_eye"); if (std::fabs(g_config.flag_threshold - next.flag_threshold) > 0.0001f) diffs.push_back("flag_threshold"); if (g_config.blink_mode != next.blink_mode) diffs.push_back("blink_mode"); if (g_config.model_path != next.model_path) diffs.push_back("model_path"); if (g_config.osc_ip != next.osc_ip) diffs.push_back("osc_ip"); if (g_config.osc_port != next.osc_port) diffs.push_back("osc_port"); if (g_config.enabled_alignment_helper != next.enabled_alignment_helper) diffs.push_back("enabled_alignment_helper"); std::string msg = "Config changed fields:"; for (const auto& diff : diffs) { msg += " " + diff; } AppendRuntimeLog(msg); g_config_log_count.fetch_add(1); } g_config = next; g_config_dirty.store(true); } } void EyeTrackingRuntime::Start() { bool expected = false; if (running.compare_exchange_strong(expected, true)) { stop_requested.store(false); std::thread([this]() { RunLoop(); }).detach(); } } void EyeTrackingRuntime::Stop() { stop_requested.store(true); running.store(false); } bool EyeTrackingRuntime::LoopRunning() const { return running.load(); } bool EyeTrackingRuntime::RuntimeActive() const { return running.load() && inference_ready.load() && camera_initialized.load(); } bool EyeTrackingRuntime::UsingDirectML() const { return using_directml.load(); } bool EyeTrackingRuntime::ConsumeDmlFallbackFlag() { return dml_fallback_triggered.exchange(false); } std::string EyeTrackingRuntime::GetLastError() const { std::lock_guard guard(state_mutex); return last_error; } void EyeTrackingRuntime::RunLoop() { ClearRuntimeLog(); void* exception_handle = AddVectoredExceptionHandler(1, EtRuntimeExceptionHandler); HRESULT coinit_hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); bool coinit_ok = SUCCEEDED(coinit_hr) || coinit_hr == RPC_E_CHANGED_MODE; if (!coinit_ok) { AppendRuntimeLog("CoInitializeEx failed: 0x" + std::to_string(static_cast(coinit_hr))); } try { EtCamera camera; EtInference inference; FrustumLoader frustum; MemmapPublisher memmap; VRCFTPublisher vrcft; VRChatPublisher vrchat; ViewerPublisher viewer; EtRuntimeConfig local_config{}; auto refresh_config = [&]() { std::lock_guard guard(g_config_mutex); local_config = g_config; }; refresh_config(); auto apply_config = [&]() { std::string error; if (local_config.enabled) { AppendRuntimeLog("ApplyConfig: enabled=true model=" + local_config.model_path); if (!camera.Initialize()) { // Initial attempt fails, start search loop camera.StartCameraSearch(); } if (!local_config.model_path.empty()) { if (!inference.LoadModel(local_config.model_path, error)) { std::lock_guard guard(state_mutex); model_failure.store(true); AppendRuntimeLog("ApplyConfig model load failed: " + error); return false; } else { model_failure.store(false); inference_ready.store(true); } if (!frustum.Load(error)) { std::lock_guard guard(state_mutex); frustum_failure.store(true); AppendRuntimeLog("ApplyConfig frustum load failed: " + error); return false; } else { frustum_failure.store(false); } } if (local_config.publish_memmap) { if (!memmap.Initialize(error)) { AppendRuntimeLog("ApplyConfig memmap init failed: " + error); } else { AppendRuntimeLog("ApplyConfig memmap init OK."); } } else { memmap.Close(); } if (local_config.publish_vrcft) { if (!vrcft.Initialize(error)) { AppendRuntimeLog("ApplyConfig VRCFT init failed: " + error); } else { AppendRuntimeLog("ApplyConfig VRCFT init OK."); } vrcft.Configure(local_config.smoothing_intensity, local_config.smoothing_enabled, local_config.blink_mode); vrcft.UpdateFrustum(frustum); } else { vrcft.Close(); } if (local_config.publish_vrchat) { if (!vrchat.Initialize(local_config.osc_ip, local_config.osc_port, error)) { AppendRuntimeLog("ApplyConfig VRChat init failed: " + error); } else { AppendRuntimeLog("ApplyConfig VRChat init OK."); } vrchat.Configure(local_config.smoothing_intensity, local_config.smoothing_enabled, local_config.blink_mode); vrchat.UpdateFrustum(frustum); } else { vrchat.Shutdown(); } if (local_config.publish_viewer) { viewer.Configure(local_config.smoothing_intensity, local_config.smoothing_enabled); } else { viewer.Close(); } using_directml.store(inference.UsingDirectML()); } else { camera.Shutdown(); vrchat.Shutdown(); viewer.Close(); vrcft.Close(); memmap.Close(); } // Manage SteamVR watch process if (local_config.enabled) { steamvrWatch.StartWatch(); } else { steamvrWatch.EndWatch(); } return true; }; apply_config(); AppendRuntimeLog("RunLoop started."); int frame_error_count = 0; int loop_log_count = 0; int tick_log_count = 0; int seh_log_count = 0; int frame_failures = 0; uint64_t last_camera_restart_ms = 0; uint64_t last_failure_log_ms = 0; uint64_t camera_cooldown_until = 0; int restart_attempts = 0; double last_heartbeat_ms = GetTicksMsec(); const uint64_t heartbeat_interval_ms = 15000; while (!stop_requested.load()) { double now_top = GetTicksMsec(); if (now_top - last_heartbeat_ms >= heartbeat_interval_ms) { last_heartbeat_ms = now_top; AppendRuntimeLog("RunLoop active."); } if (tick_log_count < 5) { AppendRuntimeLog("Loop tick."); tick_log_count++; } if (g_config_dirty.exchange(false)) { AppendRuntimeLog("Config changed, reloading runtime."); refresh_config(); apply_config(); } steamvr_waiting.store(false); if (!local_config.enabled || frustum_failure.load() || model_failure.load()) { Sleep(50); continue; } // If the user has the SteamVR only setting on, pause eyetracking loop until the process is found if (steamvrWatch.BlockFromSteamVRAbsent()) { camera.StartCameraSearch(); // Unlock camera resource and restart search steamvr_waiting.store(true); Sleep(1000); continue; } cv::Mat frame; double now = GetTicksMsec(); if (now < camera_cooldown_until) { Sleep(20); continue; } bool frame_seh = false; GetFrameWithSeh(&camera, &frame, &frame_seh); if (frame_seh || frame.empty()) { LogOnce("Frame invalid: empty frame", frame_error_count); Sleep(5); continue; } int width = frame.cols; int height = frame.rows; if (loop_log_count < 5) { AppendRuntimeLog("Frame acquired: " + std::to_string(width) + "x" + std::to_string(height)); loop_log_count++; } if (width <= 0 || height <= 0) { LogOnce("Frame invalid: width/height <= 0 (" + std::to_string(width) + "x" + std::to_string(height) + ")", frame_error_count); continue; } if ((width % 2) != 0) { LogOnce("Frame invalid: width not even (" + std::to_string(width) + ")", frame_error_count); continue; } int half_width = width / 2; cv::Mat left_eye = frame(cv::Rect(0, 0, half_width, height)); cv::Mat right_eye = frame(cv::Rect(half_width, 0, half_width, height)); bool track_left = local_config.left_eye_enabled; bool track_right = local_config.right_eye_enabled; MultiTaskPrediction left_pred{}; MultiTaskPrediction right_pred{}; if (!track_left && !track_right) { continue; } else if (track_right && !track_left) { bool seh_hit = false; if (!RunEyeWithSeh(&inference, &right_eye, true, &right_pred, &seh_hit) || seh_hit) { LogOnce("SEH during RunEye (right only).", seh_log_count); continue; } left_pred = right_pred; } else if (track_left && !track_right) { bool seh_hit = false; if (!RunEyeWithSeh(&inference, &left_eye, false, &left_pred, &seh_hit) || seh_hit) { LogOnce("SEH during RunEye (left only).", seh_log_count); continue; } right_pred = left_pred; } else { bool seh_hit = false; if (!RunEyeWithSeh(&inference, &left_eye, false, &left_pred, &seh_hit) || seh_hit) { LogOnce("SEH during RunEye (left).", seh_log_count); continue; } if (!RunEyeWithSeh(&inference, &right_eye, true, &right_pred, &seh_hit) || seh_hit) { LogOnce("SEH during RunEye (right).", seh_log_count); continue; } } EyeTrackingData data{}; data.left_x = left_pred.position[0]; data.left_y = left_pred.position[1]; data.right_x = right_pred.position[0]; data.right_y = right_pred.position[1]; data.left_flag_probability = left_pred.flag_probability; data.right_flag_probability = right_pred.flag_probability; data.timestamp_ms = static_cast(std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()).count()); data.SetFlagConditions(local_config.flag_threshold); data.right_eye_image = right_eye.clone(); data.left_eye_image = left_eye.clone(); if (local_config.publish_memmap) { bool seh_hit = false; if (!MemmapProcessWithSeh(&memmap, &data, &frustum, &seh_hit) || seh_hit) { LogOnce("SEH during MemmapPublisher::Process.", seh_log_count); continue; } } if (local_config.publish_vrcft) { bool seh_hit = false; if (!VrcftOnDataWithSeh(&vrcft, &data, &seh_hit) || seh_hit) { LogOnce("SEH during VRCFTPublisher::OnData.", seh_log_count); continue; } } if (local_config.publish_vrchat) { bool seh_hit = false; if (!VrchatOnDataWithSeh(&vrchat, &data, &seh_hit) || seh_hit) { LogOnce("SEH during VRChatPublisher::OnData.", seh_log_count); continue; } } if (local_config.publish_viewer) { bool seh_hit = false; if (!ViewerOnDataWithSeh(&viewer, &data, &seh_hit) || seh_hit) { LogOnce("SEH during Viewer::OnData.", seh_log_count); continue; } } } memmap.Close(); vrcft.Close(); viewer.Close(); vrchat.Shutdown(); camera.Shutdown(); steamvrWatch.EndWatch(); steamvr_waiting.store(false); running.store(false); } catch (const std::exception& ex) { { std::lock_guard guard(state_mutex); last_error = ex.what(); } AppendRuntimeLog(std::string("RunLoop exception: ") + ex.what()); running.store(false); } catch (...) { { std::lock_guard guard(state_mutex); last_error = "Unknown eyetracking runtime exception."; } AppendRuntimeLog("RunLoop exception: unknown."); running.store(false); } if (SUCCEEDED(coinit_hr)) { CoUninitialize(); } if (exception_handle) { RemoveVectoredExceptionHandler(exception_handle); } }