diff --git a/SteamLinkVRCFTModule/SteamLinkVRCFTModule/OSCHandler.cs b/SteamLinkVRCFTModule/SteamLinkVRCFTModule/OSCHandler.cs index f08ff49..01c667d 100644 --- a/SteamLinkVRCFTModule/SteamLinkVRCFTModule/OSCHandler.cs +++ b/SteamLinkVRCFTModule/SteamLinkVRCFTModule/OSCHandler.cs @@ -1,163 +1,169 @@ using Microsoft.Extensions.Logging; -using System; -using System.Collections.Generic; -using System.Linq; using System.Net; using System.Net.Sockets; -using System.Text; -using System.Threading.Tasks; -using VRCFaceTracking; -using VRCFaceTracking.Core.OSC; using VRCFaceTracking.Core.Params.Expressions; -using VRCFaceTracking.OSC; -using System.Buffers.Binary; -using System.Globalization; -namespace SteamLinkVRCFTModule -{ - using static UnifiedExpressions; - - public class OSCHandler - { - public readonly float[] eyeTrackData = new float[3]; - public readonly float[] eyelids = new float[2]; - - public static readonly Dictionary ueData = new(){ - {EyeWideLeft, 0.0f }, - {EyeWideRight, 0.0f }, - {EyeSquintLeft, 0.0f }, - {EyeSquintRight, 0.0f }, - {BrowInnerUpLeft, 0.0f }, - {BrowInnerUpRight, 0.0f }, - {BrowOuterUpLeft, 0.0f }, - {BrowOuterUpRight, 0.0f }, - {BrowPinchLeft, 0.0f }, - {BrowLowererLeft, 0.0f }, - {BrowLowererRight, 0.0f }, - {BrowPinchRight, 0.0f }, - - { JawOpen, 0.0f}, - { JawLeft, 0.0f}, - { JawRight, 0.0f}, - { JawForward, 0.0f}, - { MouthLowerLeft, 0.0f}, - { MouthUpperLeft, 0.0f}, - { MouthLowerRight, 0.0f}, - { MouthUpperRight, 0.0f}, - { MouthRaiserUpper, 0.0f}, - { MouthRaiserLower, 0.0f}, +namespace SteamLinkVRCFTModule; - {MouthDimpleLeft ,0.0f}, - { MouthDimpleRight,0.0f}, - { MouthClosed,0.0f}, - { MouthCornerPullLeft,0.0f}, - { MouthCornerSlantLeft,0.0f}, - { MouthCornerPullRight,0.0f}, - { MouthCornerSlantRight,0.0f}, - - {MouthFrownLeft ,0.0f}, - { MouthFrownRight,0.0f}, - { MouthLowerDownLeft,0.0f}, - {MouthLowerDownRight ,0.0f}, - { MouthUpperUpLeft,0.0f}, //TODO CHECK - {MouthUpperUpRight ,0.0f}, //TODO CHECK - {MouthTightenerLeft ,0.0f}, - { MouthTightenerRight,0.0f}, - {MouthPressLeft ,0.0f}, - {MouthPressRight ,0.0f}, - {MouthStretchLeft ,0.0f}, - {MouthStretchRight ,0.0f}, - {LipPuckerLowerLeft ,0.0f}, - { LipPuckerUpperLeft,0.0f}, - { LipPuckerLowerRight,0.0f}, - {LipPuckerUpperRight ,0.0f}, - { LipFunnelLowerLeft,0.0f}, - { LipFunnelUpperLeft,0.0f}, - { LipFunnelLowerRight,0.0f}, - { LipFunnelUpperRight,0.0f}, - {LipSuckLowerLeft,0.0f}, - { LipSuckUpperLeft,0.0f}, - {LipSuckLowerRight ,0.0f}, - { LipSuckUpperRight,0.0f}, - {CheekPuffLeft ,0.0f}, - { CheekPuffRight,0.0f}, - {CheekSuckLeft ,0.0f}, - { CheekSuckRight,0.0f}, - { CheekSquintLeft,0.0f}, - {CheekSquintRight ,0.0f}, - { NoseSneerLeft,0.0f}, - {NoseSneerRight ,0.0f}, - - {TongueCurlUp ,0.0f}, - {TongueOut ,0.0f}, - - }; +using static UnifiedExpressions; +public class OscHandler +{ + public readonly float[] EyeTrackData = new float[3]; + public readonly float[] Eyelids = new float[2]; - //based on https://docs.google.com/spreadsheets/d/118jo960co3Mgw8eREFVBsaJ7z0GtKNr52IB4Bz99VTA/edit#gid=0 - private static readonly - Dictionary> mapOSCDirectXRFBUnifiedExpressions = new Dictionary> + public static readonly Dictionary UeData = new() + { + { EyeWideLeft, 0.0f }, + { EyeWideRight, 0.0f }, + { EyeSquintLeft, 0.0f }, + { EyeSquintRight, 0.0f }, + { BrowInnerUpLeft, 0.0f }, + { BrowInnerUpRight, 0.0f }, + { BrowOuterUpLeft, 0.0f }, + { BrowOuterUpRight, 0.0f }, + { BrowPinchLeft, 0.0f }, + { BrowLowererLeft, 0.0f }, + { BrowLowererRight, 0.0f }, + { BrowPinchRight, 0.0f }, + + { JawOpen, 0.0f }, + { JawLeft, 0.0f }, + { JawRight, 0.0f }, + { JawForward, 0.0f }, + { MouthLowerLeft, 0.0f }, + { MouthUpperLeft, 0.0f }, + { MouthLowerRight, 0.0f }, + { MouthUpperRight, 0.0f }, + { MouthRaiserUpper, 0.0f }, + { MouthRaiserLower, 0.0f }, + + { MouthDimpleLeft, 0.0f }, + { MouthDimpleRight, 0.0f }, + { MouthClosed, 0.0f }, + { MouthCornerPullLeft, 0.0f }, + { MouthCornerSlantLeft, 0.0f }, + { MouthCornerPullRight, 0.0f }, + { MouthCornerSlantRight, 0.0f }, + + { MouthFrownLeft, 0.0f }, + { MouthFrownRight, 0.0f }, + { MouthLowerDownLeft, 0.0f }, + { MouthLowerDownRight, 0.0f }, + { MouthUpperUpLeft, 0.0f }, //TODO CHECK + { MouthUpperUpRight, 0.0f }, //TODO CHECK + { MouthTightenerLeft, 0.0f }, + { MouthTightenerRight, 0.0f }, + { MouthPressLeft, 0.0f }, + { MouthPressRight, 0.0f }, + { MouthStretchLeft, 0.0f }, + { MouthStretchRight, 0.0f }, + { LipPuckerLowerLeft, 0.0f }, + { LipPuckerUpperLeft, 0.0f }, + { LipPuckerLowerRight, 0.0f }, + { LipPuckerUpperRight, 0.0f }, + { LipFunnelLowerLeft, 0.0f }, + { LipFunnelUpperLeft, 0.0f }, + { LipFunnelLowerRight, 0.0f }, + { LipFunnelUpperRight, 0.0f }, + { LipSuckLowerLeft, 0.0f }, + { LipSuckUpperLeft, 0.0f }, + { LipSuckLowerRight, 0.0f }, + { LipSuckUpperRight, 0.0f }, + { CheekPuffLeft, 0.0f }, + { CheekPuffRight, 0.0f }, + { CheekSuckLeft, 0.0f }, + { CheekSuckRight, 0.0f }, + { CheekSquintLeft, 0.0f }, + { CheekSquintRight, 0.0f }, + { NoseSneerLeft, 0.0f }, + { NoseSneerRight, 0.0f }, + + { TongueCurlUp, 0.0f }, + { TongueOut, 0.0f }, + }; + + + //based on https://docs.google.com/spreadsheets/d/118jo960co3Mgw8eREFVBsaJ7z0GtKNr52IB4Bz99VTA/edit#gid=0 + private static readonly + Dictionary> MapOscDirectXrfbUnifiedExpressions = + new() { - {"/sl/xrfb/facew/UpperLidRaiserL", new List{EyeWideLeft}}, - {"/sl/xrfb/facew/UpperLidRaiserR", new List{EyeWideRight}}, - {"/sl/xrfb/facew/LidTightenerL", new List{EyeSquintLeft}}, - {"/sl/xrfb/facew/LidTightenerR", new List{EyeSquintRight}}, - {"/sl/xrfb/facew/InnerBrowRaiserL", new List{BrowInnerUpLeft}}, - {"/sl/xrfb/facew/InnerBrowRaiserR", new List{BrowInnerUpRight}}, - {"/sl/xrfb/facew/OuterBrowRaiserL", new List{BrowOuterUpLeft}}, - {"/sl/xrfb/facew/OuterBrowRaiserR", new List{BrowOuterUpRight}}, - {"/sl/xrfb/facew/BrowLowererL", new List{BrowPinchLeft, BrowLowererLeft}}, - {"/sl/xrfb/facew/BrowLowererR", new List{BrowPinchRight, BrowLowererRight}}, - - {"/sl/xrfb/facew/JawDrop", new List{JawOpen}}, - {"/sl/xrfb/facew/JawSidewaysLeft", new List{JawLeft}}, - {"/sl/xrfb/facew/JawSidewaysRight", new List{JawRight}}, - {"/sl/xrfb/facew/JawThrust", new List{JawForward}}, - - {"/sl/xrfb/facew/MouthLeft", new List{MouthLowerLeft, MouthUpperLeft}}, - {"/sl/xrfb/facew/MouthRight", new List{MouthLowerRight, MouthUpperRight}}, - - {"/sl/xrfb/facew/ChinRaiserT", new List{MouthRaiserUpper} }, - {"/sl/xrfb/facew/ChinRaiserB", new List{MouthRaiserLower} }, - - {"/sl/xrfb/facew/DimplerL", new List{MouthDimpleLeft} }, - {"/sl/xrfb/facew/DimplerR", new List{MouthDimpleRight} }, - - {"/sl/xrfb/facew/LipsToward", new List{MouthClosed}}, - {"/sl/xrfb/facew/LipCornerPullerL", new List{ MouthCornerPullLeft, MouthCornerSlantLeft} }, - {"/sl/xrfb/facew/LipCornerPullerR", new List{ MouthCornerPullRight, MouthCornerSlantRight} }, - {"/sl/xrfb/facew/LipCornerDepressorL", new List{ MouthFrownLeft} }, - {"/sl/xrfb/facew/LipCornerDepressorR", new List{ MouthFrownRight} }, - {"/sl/xrfb/facew/LowerLipDepressorL", new List{ MouthLowerDownLeft} }, - {"/sl/xrfb/facew/LowerLipDepressorR", new List{ MouthLowerDownRight} }, - {"/sl/xrfb/facew/UpperLipRaiserL", new List{ MouthUpperUpLeft} }, //something odd here - {"/sl/xrfb/facew/UpperLipRaiserR", new List{ MouthUpperUpRight } }, //something odd here - {"/sl/xrfb/facew/LipTightenerL", new List{MouthTightenerLeft} }, - {"/sl/xrfb/facew/LipTightenerR", new List{MouthTightenerRight} }, - {"/sl/xrfb/facew/LipPressorL", new List{MouthPressLeft} }, - {"/sl/xrfb/facew/LipPressorR", new List{MouthPressRight} }, - {"/sl/xrfb/facew/LipStretcherL", new List{MouthStretchLeft} }, - {"/sl/xrfb/facew/LipStretcherR", new List{MouthStretchRight} }, - {"/sl/xrfb/facew/LipPuckerL", new List{ LipPuckerLowerLeft, LipPuckerUpperLeft } }, - {"/sl/xrfb/facew/LipPuckerR", new List{ LipPuckerLowerRight, LipPuckerUpperRight } }, - {"/sl/xrfb/facew/LipFunnelerLB", new List{LipFunnelLowerLeft} }, - {"/sl/xrfb/facew/LipFunnelerLT", new List{LipFunnelUpperLeft} }, - {"/sl/xrfb/facew/LipFunnelerRB", new List{LipFunnelLowerRight} }, - {"/sl/xrfb/facew/LipFunnelerRT", new List{LipFunnelUpperRight} }, - {"/sl/xrfb/facew/LipSuckLB", new List{LipSuckLowerLeft} }, - {"/sl/xrfb/facew/LipSuckLT", new List{LipSuckUpperLeft} }, - {"/sl/xrfb/facew/LipSuckRB", new List{LipSuckLowerRight} }, - {"/sl/xrfb/facew/LipSuckRT", new List{LipSuckUpperRight} }, - - {"/sl/xrfb/facew/CheekPuffL", new List{CheekPuffLeft} }, - {"/sl/xrfb/facew/CheekPuffR", new List{CheekPuffRight} }, - {"/sl/xrfb/facew/CheekSuckL", new List{CheekSuckLeft} }, - {"/sl/xrfb/facew/CheekSuckR", new List{CheekSuckRight} }, - {"/sl/xrfb/facew/CheekRaiserL", new List{CheekSquintLeft} }, - {"/sl/xrfb/facew/CheekRaiserR", new List{CheekSquintRight} }, - - {"/sl/xrfb/facew/NoseWrinklerL", new List{NoseSneerLeft} }, - {"/sl/xrfb/facew/NoseWrinklerR", new List{NoseSneerRight} }, + { "/sl/xrfb/facew/UpperLidRaiserL", new List { EyeWideLeft } }, + { "/sl/xrfb/facew/UpperLidRaiserR", new List { EyeWideRight } }, + { "/sl/xrfb/facew/LidTightenerL", new List { EyeSquintLeft } }, + { "/sl/xrfb/facew/LidTightenerR", new List { EyeSquintRight } }, + { "/sl/xrfb/facew/InnerBrowRaiserL", new List { BrowInnerUpLeft } }, + { "/sl/xrfb/facew/InnerBrowRaiserR", new List { BrowInnerUpRight } }, + { "/sl/xrfb/facew/OuterBrowRaiserL", new List { BrowOuterUpLeft } }, + { "/sl/xrfb/facew/OuterBrowRaiserR", new List { BrowOuterUpRight } }, + { "/sl/xrfb/facew/BrowLowererL", new List { BrowPinchLeft, BrowLowererLeft } }, + { "/sl/xrfb/facew/BrowLowererR", new List { BrowPinchRight, BrowLowererRight } }, + + { "/sl/xrfb/facew/JawDrop", new List { JawOpen } }, + { "/sl/xrfb/facew/JawSidewaysLeft", new List { JawLeft } }, + { "/sl/xrfb/facew/JawSidewaysRight", new List { JawRight } }, + { "/sl/xrfb/facew/JawThrust", new List { JawForward } }, + + { "/sl/xrfb/facew/MouthLeft", new List { MouthLowerLeft, MouthUpperLeft } }, + { "/sl/xrfb/facew/MouthRight", new List { MouthLowerRight, MouthUpperRight } }, + + { "/sl/xrfb/facew/ChinRaiserT", new List { MouthRaiserUpper } }, + { "/sl/xrfb/facew/ChinRaiserB", new List { MouthRaiserLower } }, + + { "/sl/xrfb/facew/DimplerL", new List { MouthDimpleLeft } }, + { "/sl/xrfb/facew/DimplerR", new List { MouthDimpleRight } }, + + { "/sl/xrfb/facew/LipsToward", new List { MouthClosed } }, + { + "/sl/xrfb/facew/LipCornerPullerL", + new List { MouthCornerPullLeft, MouthCornerSlantLeft } + }, + { + "/sl/xrfb/facew/LipCornerPullerR", + new List { MouthCornerPullRight, MouthCornerSlantRight } + }, + { "/sl/xrfb/facew/LipCornerDepressorL", new List { MouthFrownLeft } }, + { "/sl/xrfb/facew/LipCornerDepressorR", new List { MouthFrownRight } }, + { "/sl/xrfb/facew/LowerLipDepressorL", new List { MouthLowerDownLeft } }, + { "/sl/xrfb/facew/LowerLipDepressorR", new List { MouthLowerDownRight } }, + { + "/sl/xrfb/facew/UpperLipRaiserL", new List { MouthUpperUpLeft } + }, //something odd here + { + "/sl/xrfb/facew/UpperLipRaiserR", new List { MouthUpperUpRight } + }, //something odd here + { "/sl/xrfb/facew/LipTightenerL", new List { MouthTightenerLeft } }, + { "/sl/xrfb/facew/LipTightenerR", new List { MouthTightenerRight } }, + { "/sl/xrfb/facew/LipPressorL", new List { MouthPressLeft } }, + { "/sl/xrfb/facew/LipPressorR", new List { MouthPressRight } }, + { "/sl/xrfb/facew/LipStretcherL", new List { MouthStretchLeft } }, + { "/sl/xrfb/facew/LipStretcherR", new List { MouthStretchRight } }, + { + "/sl/xrfb/facew/LipPuckerL", new List { LipPuckerLowerLeft, LipPuckerUpperLeft } + }, + { + "/sl/xrfb/facew/LipPuckerR", + new List { LipPuckerLowerRight, LipPuckerUpperRight } + }, + { "/sl/xrfb/facew/LipFunnelerLB", new List { LipFunnelLowerLeft } }, + { "/sl/xrfb/facew/LipFunnelerLT", new List { LipFunnelUpperLeft } }, + { "/sl/xrfb/facew/LipFunnelerRB", new List { LipFunnelLowerRight } }, + { "/sl/xrfb/facew/LipFunnelerRT", new List { LipFunnelUpperRight } }, + { "/sl/xrfb/facew/LipSuckLB", new List { LipSuckLowerLeft } }, + { "/sl/xrfb/facew/LipSuckLT", new List { LipSuckUpperLeft } }, + { "/sl/xrfb/facew/LipSuckRB", new List { LipSuckLowerRight } }, + { "/sl/xrfb/facew/LipSuckRT", new List { LipSuckUpperRight } }, + + { "/sl/xrfb/facew/CheekPuffL", new List { CheekPuffLeft } }, + { "/sl/xrfb/facew/CheekPuffR", new List { CheekPuffRight } }, + { "/sl/xrfb/facew/CheekSuckL", new List { CheekSuckLeft } }, + { "/sl/xrfb/facew/CheekSuckR", new List { CheekSuckRight } }, + { "/sl/xrfb/facew/CheekRaiserL", new List { CheekSquintLeft } }, + { "/sl/xrfb/facew/CheekRaiserR", new List { CheekSquintRight } }, + + { "/sl/xrfb/facew/NoseWrinklerL", new List { NoseSneerLeft } }, + { "/sl/xrfb/facew/NoseWrinklerR", new List { NoseSneerRight } }, //Tongue Tracking Params //Some params sent but not part of UE or used? need to look into @@ -167,140 +173,142 @@ private static readonly //{"/sl/xrfb/facew/BackDorsalVelar", new List{} }, //{"/sl/xrfb/facew/TongueRetreat", new List{} }, //{"/sl/xrfb/facew/TongueTipAlveolar", new List{TongueCurlUp} }, - {"/sl/xrfb/facew/TongueOut", new List{TongueOut} }, + { "/sl/xrfb/facew/TongueOut", new List { TongueOut } }, }; + private Socket _receiver; + private bool _loop = true; + private readonly Thread _thread; + private readonly ILogger _logger; + private readonly int _resolvedPort; + private const int DefaultPort = 9015; + private const int TimeoutMs = 10_000; - private Socket _receiver; - private bool _loop = true; - private readonly Thread _thread; - private readonly ILogger _logger; - private readonly int _resolvedPort; - private const int DEFAULT_PORT = 9015; - private const int TIMEOUT_MS = 10_000; - - public OSCHandler(ILogger iLogger, int? port = null) + public OscHandler(ILogger iLogger, int? port = null) + { + _logger = iLogger; + if (_receiver != null) { - _logger = iLogger; - if (_receiver != null) - { - _logger.LogError("OSCHandler connection already exists."); - return; - } - - _receiver = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); - _resolvedPort = port ?? DEFAULT_PORT; - _receiver.Bind(new IPEndPoint(IPAddress.Parse("127.0.0.1"), _resolvedPort)); - _receiver.ReceiveTimeout = TIMEOUT_MS; - - _loop = true; - _thread = new Thread(new ThreadStart(ListenLoop)); - _thread.Start(); - // 1 second delay to hopefully fix any race conditions on thread initalization - //TODO remove this and actually fix the issue - Thread.Sleep(1000); + _logger.LogError("OSCHandler connection already exists."); + return; } - private void ListenLoop() + _receiver = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + _resolvedPort = port ?? DefaultPort; + _receiver.Bind(new IPEndPoint(IPAddress.Parse("127.0.0.1"), _resolvedPort)); + _receiver.ReceiveTimeout = TimeoutMs; + + _loop = true; + _thread = new Thread(ListenLoop); + _thread.Start(); + // 1 second delay to hopefully fix any race conditions on thread initialization + //TODO remove this and actually fix the issue + Thread.Sleep(1000); + } + + private void ListenLoop() + { + var buffer = new byte[8192]; + while (_loop) { - var buffer = new byte[8192]; - while (_loop) + try { - try + if (_receiver.IsBound) { - if (_receiver.IsBound) + var length = _receiver.Receive(buffer); + var msgList = new List(); + if (OscParser.IsBundle(ref buffer)) { + var i = 16; + var elLength = new byte[4]; + while (i < length) + { + var messageLength = 0; + if (BitConverter.IsLittleEndian) + { + var msgsize = new byte[4]; + msgsize[0] = buffer[i]; + msgsize[1] = buffer[i + 1]; + msgsize[2] = buffer[i + 2]; + msgsize[3] = buffer[i + 3]; + Array.Reverse(msgsize); + messageLength = BitConverter.ToInt32(msgsize, 0); + } + else + { + messageLength = BitConverter.ToInt32(buffer, i); + } + var adjustLength = 4 - messageLength % 4; + var temp = new byte[messageLength]; + Array.Copy(buffer, i + 4, temp, 0, messageLength); + msgList.Add(new Oscm(ref temp, _logger)); + i = i + messageLength + adjustLength; + } + } + else + { + msgList.Add(new Oscm(ref buffer, _logger)); + } - - var length = _receiver.Receive(buffer); - List msgList = new List(); - if (OSCParser.IsBundle(ref buffer)) + foreach (var oscMessage in msgList) + { + if (oscMessage == null) continue; + if (oscMessage.Values.Count < 1) continue; + if (oscMessage.Address == "/sl/eyeTrackedGazePoint") { - int i = 16; - var elLength = new byte[4]; - while (i < length) + for (var i = 0; i < 3; i++) { - int messageLength = 0; - if (BitConverter.IsLittleEndian) - { - byte[] msgsize = new byte[4]; - msgsize[0] = buffer[i]; msgsize[1] = buffer[i + 1]; msgsize[2] = buffer[i + 2]; msgsize[3] = buffer[i + 3]; - Array.Reverse(msgsize); - messageLength = BitConverter.ToInt32(msgsize, 0); - } - else - { - messageLength = BitConverter.ToInt32(buffer, i); - } - int adjustLength = 4 - messageLength % 4; - byte[] temp = new byte[messageLength]; - Array.Copy(buffer, i + 4, temp, 0, messageLength); - msgList.Add(new OSCM(ref temp, _logger)); - i = i + messageLength + adjustLength; - + EyeTrackData[i] = (float)oscMessage.Values[i]; } + + continue; } - else + + if (oscMessage.Address == ("/sl/xrfb/facew/EyesClosedL")) { - msgList.Add(new OSCM(ref buffer, _logger)); + Eyelids[0] = (float)oscMessage.Values[0]; + continue; } - foreach (OSCM oscMessage in msgList) - { - if (oscMessage == null) continue; - if (oscMessage.Values.Count < 1) continue; - if (oscMessage.Address == "/sl/eyeTrackedGazePoint") - { - for (int i = 0; i < 3; i++) - { - eyeTrackData[i] = (float)oscMessage.Values[i]; - } - continue; - } - if (oscMessage.Address == ("/sl/xrfb/facew/EyesClosedL")) - { - eyelids[0] = (float)oscMessage.Values[0]; - continue; - } - if (oscMessage.Address == "/sl/xrfb/facew/EyesClosedR") - { - eyelids[1] = (float)oscMessage.Values[0]; - continue; - } + if (oscMessage.Address == "/sl/xrfb/facew/EyesClosedR") + { + Eyelids[1] = (float)oscMessage.Values[0]; + continue; + } - if (mapOSCDirectXRFBUnifiedExpressions.ContainsKey(oscMessage.Address)) + if (MapOscDirectXrfbUnifiedExpressions.ContainsKey(oscMessage.Address)) + { + foreach (var unifiedExpression in MapOscDirectXrfbUnifiedExpressions[oscMessage.Address]) { - foreach (UnifiedExpressions unifiedExpression in mapOSCDirectXRFBUnifiedExpressions[oscMessage.Address]) - { - //This may not be strictly safe but should be good enough for our use case - ueData[unifiedExpression] = (float)oscMessage.Values[0]; } + //This may not be strictly safe but should be good enough for our use case + UeData[unifiedExpression] = (float)oscMessage.Values[0]; } } } - else - { - _receiver.Close(); - _receiver.Dispose(); - _receiver = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); - _receiver.Bind(new IPEndPoint(IPAddress.Parse("127.0.0.1"), _resolvedPort)); - _receiver.ReceiveTimeout = TIMEOUT_MS; - } } - catch (Exception e) { - _logger.LogInformation("Exception {0}", e.Message); + else + { + _receiver.Close(); + _receiver.Dispose(); + _receiver = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + _receiver.Bind(new IPEndPoint(IPAddress.Parse("127.0.0.1"), _resolvedPort)); + _receiver.ReceiveTimeout = TimeoutMs; } } + catch (Exception e) + { + _logger.LogInformation("Exception {0}", e.Message); + } } + } - public void Teardown() - { - _loop = false; - _receiver.Close(); - _receiver.Dispose(); - _thread.Join(); - } - + public void Teardown() + { + _loop = false; + _receiver.Close(); + _receiver.Dispose(); + _thread.Join(); } -} +} \ No newline at end of file diff --git a/SteamLinkVRCFTModule/SteamLinkVRCFTModule/OSCParser.cs b/SteamLinkVRCFTModule/SteamLinkVRCFTModule/OSCParser.cs index 210ee21..37beb7b 100644 --- a/SteamLinkVRCFTModule/SteamLinkVRCFTModule/OSCParser.cs +++ b/SteamLinkVRCFTModule/SteamLinkVRCFTModule/OSCParser.cs @@ -1,211 +1,222 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; -using System.Net.Sockets; +using System.Collections; using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Microsoft.VisualBasic; -using VRCFaceTracking.Core.OSC; -namespace SteamLinkVRCFTModule +namespace SteamLinkVRCFTModule; + +public class Oscm { - public class OSCM - { - public string Address = ""; - public ArrayList Values = new ArrayList(); - //int = 0, float = 1, blob = 2, string = 3, error = -1 - private List valType = new List(); + public string Address = ""; + + public readonly ArrayList Values = new(); + //int = 0, float = 1, blob = 2, string = 3, error = -1 + private readonly List _valType = new(); - public int getAddress(ref byte[] msg, int index) + private int GetAddress(ref byte[] msg, int index) + { + while (index < msg.Length) { - while (index < msg.Length) + if (msg[index] == 0x2c) { - if (msg[index] == 0x2c) - { - //Address = Encoding.ASCII.GetString(msg, 0, index - 1); - index++; - return index; - } - //This is sloppy, But need to verify if OSC spec is one null char on address or if it's up to 4? - if (msg[index] == 0x00) - { - if(Address == "") - { - Address = Encoding.ASCII.GetString(msg, 0, index); - } - } - + //Address = Encoding.ASCII.GetString(msg, 0, index - 1); index++; + return index; } - return -1; - } - public int getParams(ref byte[] msg, int index, ILogger log) - { - int valmax = 0; - //we start at the parameter not the "," in an osc string, and then we need - // an additional to account for offset - int typetagstart = index-1; - while (index < msg.Length) + //This is sloppy, But need to verify if OSC spec is one null char on address or if it's up to 4? + if (msg[index] == 0x00) { - switch (msg[index]) + if (Address == "") { - //null i.e. end of params - case 0x00: - return (4 - ((index - typetagstart) % 4 )+ index); - //float - case 0x66: - valType.Add(1); - valmax++; - break; - //int - case 0x69: - valType.Add(0); - valmax++; - break; - //blob - case 0x62: - valType.Add(2); - valmax++; - break; - //string - case 0x72: - valType.Add(3); - valmax++; - break; - default: - valType.Add(-1); - break; + Address = Encoding.ASCII.GetString(msg, 0, index); } - index++; } - return -1; + + index++; } - public int getValues(ref byte[] message, int i, ILogger log) - { - int valuecount = 0; - int maxVal = valType.Count(); - while (i < message.Length) - { - byte[] msgsize = new byte[4]; - switch (valType[valuecount]) - { - case 0: - if (BitConverter.IsLittleEndian) - { - msgsize[0] = message[i]; msgsize[1] = message[i + 1]; msgsize[2] = message[i + 2]; msgsize[3] = message[i + 3]; - Array.Reverse(msgsize); - Values.Add(BitConverter.ToInt32(msgsize, 0)); - } - else - { - Values.Add(BitConverter.ToInt32(message, i)); - } - i = i + 3; - valuecount++; - - break; - case 1: - if (BitConverter.IsLittleEndian) - { - msgsize[0] = message[i]; msgsize[1] = message[i + 1]; msgsize[2] = message[i + 2]; msgsize[3] = message[i + 3]; - Array.Reverse(msgsize); - Values.Add(BitConverter.ToSingle(msgsize, 0)); - } - else - { - Values.Add(BitConverter.ToSingle(message, i)); - } - i = i + 3; - valuecount++; - - break; - case 2: - //TODO yea not happening (blob implementation) - break; - case 3: - log.LogInformation("string"); - int initialI = i; - while (message[i] != 0x00) - { - i++; - } - Values.Add(Encoding.ASCII.GetString(message, initialI, i)); - valuecount++; - //OSC padding to 32 bit chunks (4 byte) - i = i + ((i - initialI) % 4); - //i++; - - break; - default: - log.LogInformation("error default XXXXXXXX"); - //TODO error handling - break; + return -1; + } - } - if (valuecount == maxVal) - { - return message.Length; - } - i++; + public int GetParams(ref byte[] msg, int index, ILogger log) + { + var valmax = 0; + //we start at the parameter not the "," in an osc string, and then we need + // an additional to account for offset + var typetagstart = index - 1; + while (index < msg.Length) + { + switch (msg[index]) + { + //null i.e. end of params + case 0x00: + return (4 - ((index - typetagstart) % 4) + index); + //float + case 0x66: + _valType.Add(1); + valmax++; + break; + //int + case 0x69: + _valType.Add(0); + valmax++; + break; + //blob + case 0x62: + _valType.Add(2); + valmax++; + break; + //string + case 0x72: + _valType.Add(3); + valmax++; + break; + default: + _valType.Add(-1); + break; } - return -1; + + index++; } - public OSCM(ref byte[] message, ILogger iLogger) + return -1; + } + + private int GetValues(ref byte[] message, int i, ILogger log) + { + var valuecount = 0; + var maxVal = _valType.Count(); + while (i < message.Length) { - int i = 0; - i = getAddress(ref message, i); - if (i == -1) - { - iLogger.LogInformation("fail at addr"); - return; - } - i = getParams(ref message, i, iLogger); - if (i == -1) + var msgsize = new byte[4]; + switch (_valType[valuecount]) { - iLogger.LogInformation("fail at param"); - return; + case 0: + if (BitConverter.IsLittleEndian) + { + msgsize[0] = message[i]; + msgsize[1] = message[i + 1]; + msgsize[2] = message[i + 2]; + msgsize[3] = message[i + 3]; + Array.Reverse(msgsize); + Values.Add(BitConverter.ToInt32(msgsize, 0)); + } + else + { + Values.Add(BitConverter.ToInt32(message, i)); + } + + i = i + 3; + valuecount++; + + break; + case 1: + if (BitConverter.IsLittleEndian) + { + msgsize[0] = message[i]; + msgsize[1] = message[i + 1]; + msgsize[2] = message[i + 2]; + msgsize[3] = message[i + 3]; + Array.Reverse(msgsize); + Values.Add(BitConverter.ToSingle(msgsize, 0)); + } + else + { + Values.Add(BitConverter.ToSingle(message, i)); + } + + i = i + 3; + valuecount++; + + break; + case 2: + //TODO yea not happening (blob implementation) + break; + case 3: + log.LogInformation("string"); + var initialI = i; + while (message[i] != 0x00) + { + i++; + } + + Values.Add(Encoding.ASCII.GetString(message, initialI, i)); + valuecount++; + //OSC padding to 32 bit chunks (4 byte) + i = i + ((i - initialI) % 4); + //i++; + + break; + default: + log.LogInformation("error default XXXXXXXX"); + //TODO error handling + break; } - i = getValues(ref message, i, iLogger); - if (i == -1) + + if (valuecount == maxVal) { - iLogger.LogInformation("fail at value"); - return; + return message.Length; } + + i++; } + return -1; } - static public class OSCParser + public Oscm(ref byte[] message, ILogger iLogger) { - static readonly byte[] bufASCII = Encoding.ASCII.GetBytes("#bundle"); - public static bool IsBundle(ref byte[] buff) + var i = 0; + i = GetAddress(ref message, i); + if (i == -1) { - if(buff == null|| buff.Length<8) - { - return false; - } - byte[] bundletest = new byte[7]; - Array.Copy(buff, bundletest, 7); - return Enumerable.SequenceEqual(bundletest, bufASCII); + iLogger.LogInformation("fail at addr"); + return; } - public static uint swapEndianness(uint x) + + i = GetParams(ref message, i, iLogger); + if (i == -1) { - return ((x & 0x000000ff) << 24) + // First byte - ((x & 0x0000ff00) << 8) + // Second byte - ((x & 0x00ff0000) >> 8) + // Third byte - ((x & 0xff000000) >> 24); // Fourth byte + iLogger.LogInformation("fail at param"); + return; } - public static int trueMod(int a, int b) + + i = GetValues(ref message, i, iLogger); + if (i == -1) { - return (Math.Abs(a * b) + a) % b; + iLogger.LogInformation("fail at value"); + return; } - } } + +public static class OscParser +{ + private static readonly byte[] BufAscii = "#bundle"u8.ToArray(); + + public static bool IsBundle(ref byte[] buff) + { + if (buff == null || buff.Length < 8) + { + return false; + } + + var bundletest = new byte[7]; + Array.Copy(buff, bundletest, 7); + return bundletest.SequenceEqual(BufAscii); + } + + public static uint SwapEndianness(uint x) + { + return ((x & 0x000000ff) << 24) + // First byte + ((x & 0x0000ff00) << 8) + // Second byte + ((x & 0x00ff0000) >> 8) + // Third byte + ((x & 0xff000000) >> 24); // Fourth byte + } + + public static int TrueMod(int a, int b) + { + return (Math.Abs(a * b) + a) % b; + } +} \ No newline at end of file diff --git a/SteamLinkVRCFTModule/SteamLinkVRCFTModule/SteamLinkVRCFTModule.cs b/SteamLinkVRCFTModule/SteamLinkVRCFTModule/SteamLinkVRCFTModule.cs index ceb9255..74aea6d 100644 --- a/SteamLinkVRCFTModule/SteamLinkVRCFTModule/SteamLinkVRCFTModule.cs +++ b/SteamLinkVRCFTModule/SteamLinkVRCFTModule/SteamLinkVRCFTModule.cs @@ -1,105 +1,109 @@ -using Microsoft.Extensions.Logging; -using SteamLinkVRCFTModule; -using System.Net.Sockets; -using VRCFaceTracking; +using VRCFaceTracking; using VRCFaceTracking.Core.Params.Expressions; using static VRCFaceTracking.Core.Params.Expressions.UnifiedExpressions; -namespace SteamLinkVRCFTModule +namespace SteamLinkVRCFTModule; + +public class SteamLinkVrcftModule : ExtTrackingModule { - public class SteamLinkVRCFTModule : ExtTrackingModule - { - private OSCHandler OSCHandler; - private const int DEFAULT_PORT = 9015; + private OscHandler _oscHandler; + private const int DefaultPort = 9015; - public override (bool SupportsEye, bool SupportsExpression) Supported => (true, true); + public override (bool SupportsEye, bool SupportsExpression) Supported => (true, true); - public override (bool eyeSuccess, bool expressionSuccess) Initialize(bool eyeAvailable, bool expressionAvailable) - { - ModuleInformation.Name = "SteamLink Module"; + public override (bool eyeSuccess, bool expressionSuccess) Initialize(bool eyeAvailable, bool expressionAvailable) + { + ModuleInformation.Name = "SteamLink Module"; - var stream = System.Reflection.Assembly.GetExecutingAssembly().GetManifestResourceStream("SteamLinkVRCFTModule.Assets.steamlink.png"); - ModuleInformation.StaticImages = stream != null ? new List { stream } : ModuleInformation.StaticImages; + var stream = System.Reflection.Assembly.GetExecutingAssembly() + .GetManifestResourceStream("SteamLinkVRCFTModule.Assets.steamlink.png"); + ModuleInformation.StaticImages = stream != null ? new List { stream } : ModuleInformation.StaticImages; - //TODO better error handling on fail? isInit for OSC Handler? - OSCHandler = new OSCHandler(Logger, DEFAULT_PORT); + //TODO better error handling on fail? isInit for OSC Handler? + _oscHandler = new OscHandler(Logger, DefaultPort); - return (true, true); - } + return (true, true); + } - private static float CalculateEyeOpenness(float fEyeClosedWeight, float fEyeTightener) - { - return 1.0f - Math.Clamp(fEyeClosedWeight + fEyeClosedWeight * fEyeTightener, 0.0f, 1.0f); - } + private static float CalculateEyeOpenness(float fEyeClosedWeight, float fEyeTightener) + { + return 1.0f - Math.Clamp(fEyeClosedWeight + fEyeClosedWeight * fEyeTightener, 0.0f, 1.0f); + } - private void UpdateEyeTracking() + private void UpdateEyeTracking() + { { - { - float fAngleX = MathF.Atan2(OSCHandler.eyeTrackData[0], -OSCHandler.eyeTrackData[2]); - float fAngleY = MathF.Atan2(OSCHandler.eyeTrackData[1], -OSCHandler.eyeTrackData[2]); - - float fNmAngleX = fAngleX / (MathF.PI / 2.0f) * 2.0f; - float fNmAngleY = fAngleY / (MathF.PI / 2.0f) * 2.0f; - - if (float.IsNaN(fNmAngleX)) - { - fNmAngleX = 0.0f; - } - if (float.IsNaN(fNmAngleY)) - { - fNmAngleY = 0.0f; - } - - UnifiedTracking.Data.Eye.Left.Gaze.x = fAngleX; - UnifiedTracking.Data.Eye.Left.Gaze.y = fAngleY; - - UnifiedTracking.Data.Eye.Right.Gaze.x = fAngleX; - UnifiedTracking.Data.Eye.Right.Gaze.y = fAngleY; - - - //Pupil Dilation, This is not supported, but if we don't set it can cause issues - UnifiedTracking.Data.Eye.Left.PupilDiameter_MM = 5f; - UnifiedTracking.Data.Eye.Right.PupilDiameter_MM = 5f; - UnifiedTracking.Data.Eye._maxDilation = 10; - UnifiedTracking.Data.Eye._minDilation = 0; + var fAngleX = MathF.Atan2(_oscHandler.EyeTrackData[0], -_oscHandler.EyeTrackData[2]); + var fAngleY = MathF.Atan2(_oscHandler.EyeTrackData[1], -_oscHandler.EyeTrackData[2]); - } + var fNmAngleX = fAngleX / (MathF.PI / 2.0f) * 2.0f; + var fNmAngleY = fAngleY / (MathF.PI / 2.0f) * 2.0f; + if (float.IsNaN(fNmAngleX)) { - float fLeftOpenness = CalculateEyeOpenness(OSCHandler.eyelids[0], OSCHandler.ueData[UnifiedExpressions.EyeSquintLeft]); - float fRightOpenness = CalculateEyeOpenness(OSCHandler.eyelids[1], OSCHandler.ueData[UnifiedExpressions.EyeSquintRight]); - - UnifiedTracking.Data.Eye.Left.Openness = fLeftOpenness;// fLeftOpenness; - UnifiedTracking.Data.Eye.Right.Openness = fRightOpenness;//fRightOpenness; + fNmAngleX = 0.0f; } - } - private void UpdateFaceTracking() - { - foreach (KeyValuePair entry in OSCHandler.ueData) + if (float.IsNaN(fNmAngleY)) { - UnifiedTracking.Data.Shapes[(int)entry.Key].Weight = entry.Value; + fNmAngleY = 0.0f; } - //?? fix some weird ft things need to check (Its not needed anymore, this in fact causes upper lip to not move properly, its more notorious when you try to set an "eww" face) - //UnifiedTracking.Data.Shapes[(int)MouthUpperUpLeft].Weight = Math.Max(0, UnifiedTracking.Data.Shapes[(int)MouthUpperUpLeft].Weight - UnifiedTracking.Data.Shapes[(int)NoseSneerLeft].Weight); - //UnifiedTracking.Data.Shapes[(int)MouthUpperUpRight].Weight = Math.Max(0, UnifiedTracking.Data.Shapes[(int)MouthUpperUpRight].Weight - UnifiedTracking.Data.Shapes[(int)NoseSneerRight].Weight); - //UnifiedTracking.Data.Shapes[(int)MouthUpperDeepenLeft].Weight = Math.Max(0, UnifiedTracking.Data.Shapes[(int)MouthUpperUpLeft].Weight - UnifiedTracking.Data.Shapes[(int)NoseSneerLeft].Weight); - //UnifiedTracking.Data.Shapes[(int)MouthUpperDeepenRight].Weight = Math.Max(0, UnifiedTracking.Data.Shapes[(int)MouthUpperUpRight].Weight - UnifiedTracking.Data.Shapes[(int)NoseSneerRight].Weight); - - //lip Suck - UnifiedTracking.Data.Shapes[(int)LipSuckUpperLeft].Weight = Math.Min(1.0f - (float)Math.Pow(UnifiedTracking.Data.Shapes[(int)MouthUpperLeft].Weight, 1f / 6f), UnifiedTracking.Data.Shapes[(int)LipSuckUpperLeft].Weight); - UnifiedTracking.Data.Shapes[(int)LipSuckUpperRight].Weight = Math.Min(1.0f - (float)Math.Pow(UnifiedTracking.Data.Shapes[(int)MouthUpperRight].Weight, 1f / 6f), UnifiedTracking.Data.Shapes[(int)LipSuckUpperRight].Weight); + + UnifiedTracking.Data.Eye.Left.Gaze.x = fAngleX; + UnifiedTracking.Data.Eye.Left.Gaze.y = fAngleY; + + UnifiedTracking.Data.Eye.Right.Gaze.x = fAngleX; + UnifiedTracking.Data.Eye.Right.Gaze.y = fAngleY; + + + //Pupil Dilation, This is not supported, but if we don't set it can cause issues + UnifiedTracking.Data.Eye.Left.PupilDiameter_MM = 5f; + UnifiedTracking.Data.Eye.Right.PupilDiameter_MM = 5f; + UnifiedTracking.Data.Eye._maxDilation = 10; + UnifiedTracking.Data.Eye._minDilation = 0; } - public override void Update() { - Thread.Sleep(10); - UpdateEyeTracking(); - UpdateFaceTracking(); + var fLeftOpenness = CalculateEyeOpenness(_oscHandler.Eyelids[0], + OscHandler.UeData[EyeSquintLeft]); + var fRightOpenness = CalculateEyeOpenness(_oscHandler.Eyelids[1], + OscHandler.UeData[EyeSquintRight]); + + UnifiedTracking.Data.Eye.Left.Openness = fLeftOpenness; // fLeftOpenness; + UnifiedTracking.Data.Eye.Right.Openness = fRightOpenness; // fRightOpenness; } - public override void Teardown() + } + + private void UpdateFaceTracking() + { + foreach (var entry in OscHandler.UeData) { - OSCHandler.Teardown(); + UnifiedTracking.Data.Shapes[(int)entry.Key].Weight = entry.Value; } + //?? fix some weird ft things need to check (Its not needed anymore, this in fact causes upper lip to not move properly, its more notorious when you try to set an "eww" face) + //UnifiedTracking.Data.Shapes[(int)MouthUpperUpLeft].Weight = Math.Max(0, UnifiedTracking.Data.Shapes[(int)MouthUpperUpLeft].Weight - UnifiedTracking.Data.Shapes[(int)NoseSneerLeft].Weight); + //UnifiedTracking.Data.Shapes[(int)MouthUpperUpRight].Weight = Math.Max(0, UnifiedTracking.Data.Shapes[(int)MouthUpperUpRight].Weight - UnifiedTracking.Data.Shapes[(int)NoseSneerRight].Weight); + //UnifiedTracking.Data.Shapes[(int)MouthUpperDeepenLeft].Weight = Math.Max(0, UnifiedTracking.Data.Shapes[(int)MouthUpperUpLeft].Weight - UnifiedTracking.Data.Shapes[(int)NoseSneerLeft].Weight); + //UnifiedTracking.Data.Shapes[(int)MouthUpperDeepenRight].Weight = Math.Max(0, UnifiedTracking.Data.Shapes[(int)MouthUpperUpRight].Weight - UnifiedTracking.Data.Shapes[(int)NoseSneerRight].Weight); + + //lip Suck + UnifiedTracking.Data.Shapes[(int)LipSuckUpperLeft].Weight = Math.Min( + 1.0f - (float)Math.Pow(UnifiedTracking.Data.Shapes[(int)MouthUpperLeft].Weight, 1f / 6f), + UnifiedTracking.Data.Shapes[(int)LipSuckUpperLeft].Weight); + UnifiedTracking.Data.Shapes[(int)LipSuckUpperRight].Weight = Math.Min( + 1.0f - (float)Math.Pow(UnifiedTracking.Data.Shapes[(int)MouthUpperRight].Weight, 1f / 6f), + UnifiedTracking.Data.Shapes[(int)LipSuckUpperRight].Weight); + } + + public override void Update() + { + Thread.Sleep(10); + UpdateEyeTracking(); + UpdateFaceTracking(); + } + + public override void Teardown() + { + _oscHandler.Teardown(); } -} +} \ No newline at end of file