commit 74d66bddf5b8077b419b0e59bfae53eb329bf4f8 Author: Tony Klink Date: Thu Jun 6 22:40:08 2024 -0600 Initial commit diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..3550a30 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d8618f --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +# Generated by Cargo +# will have compiled files and executables +build/ +debug/ +target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock +godot-live-link-face.zip + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb + +# RustRover +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..013ab36 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "godot-live-link-face" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type=["cdylib"] + +[dependencies] +live-link-face = { git = "https://git.zhitno.st/klink/live-link-face" } +godot = { git = "https://github.com/godot-rust/gdext", rev = "2c5f8ed" } diff --git a/LICENSE-AGPLv3+NIGGER b/LICENSE-AGPLv3+NIGGER new file mode 100644 index 0000000..6345d61 --- /dev/null +++ b/LICENSE-AGPLv3+NIGGER @@ -0,0 +1,8 @@ +Copyright (C) 2024 Klink +This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, version 3. + +The above copyright notice, this permission notice and the word "NIGGER" shall be included in all copies or substantial portions of the Software. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License along with this program. If not, see \ No newline at end of file diff --git a/LICENSE-COMMERCIAL b/LICENSE-COMMERCIAL new file mode 100644 index 0000000..978fab2 --- /dev/null +++ b/LICENSE-COMMERCIAL @@ -0,0 +1,3 @@ +Copyright (C) 2024 Klink + +You can buy this software to be excluded from AGPLv3+NIGGER obligations \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..332a93f --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +build: + cargo build --release + cargo build --target x86_64-pc-windows-gnu --release + cp godot-live-link-face.gdextension target/release + cp -R addons target/release + cp target/release/libgodot_live_link_face.so target/release/addons/godot_live_link_face + cp target/x86_64-pc-windows-gnu/release/godot_live_link_face.dll target/release/addons/godot_live_link_face + mkdir build + mkdir -p build/addons/godot_live_link_face + mv target/release/addons/* build/addons/ + cp README.md build/ + cp LICENSE-AGPLv3+NIGGER build/ + cp LICENSE-COMMERCIAL build/ + mv target/release/godot-live-link-face.gdextension build/ + zip -r godot-live-link-face.zip build/* +clean: + rm -r build diff --git a/README.md b/README.md new file mode 100644 index 0000000..8776a2c --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# Godot LiveLinkFace + +This Godot extension allows user to connect Unreal LiveLinkFace iPhone app to the engine and control various parameters of nodes. + +## Available nodes + +### LiveLinkServer +This is a main node that handles incoming connections from the app. +Set "Listen IP" and "Listen Port" parameters to the ones set in the app configuration "UPD server" +If you have multiple sources of face data - add same number of LiveLinkServer nodes to the scene and set different listen port for each. + +Other nodes depend on LiveLinkServer to be present in the scene + +### FaceBone +This node allows to control bones of the mesh. +#### Parameters +- Read From: LiveLinkServer node to read incoming messages from +- Read Shape: Select which shape should be tracked by this node. (Ex. HeadYaw, JawOpen, etc.) +- Target Skeleton: Set the skeleton that should be controled by this node +- Target Bone: String name of the controled bone +- Clamp in Min and Clamp in Max: clamp incoming shape value to the specific range +- Clamp out Min and Clamp out Max: clamp final transform value (can be used to limit final transform range of the bone) +- Transform Axis: Which bone axis this node controls + +### FaceData +This node allows to control arbitary position of Node3d +#### Parameters +- Read From: LiveLinkServer node to read incoming messages from +- Read Shape: Select which shape should be tracked by this node. (Ex. HeadYaw, JawOpen, etc.) +- Target: Set the Node3d that should be controled by this node +- Clamp in Min and Clamp in Max: clamp incoming shape value to the specific range +- Clamp out Min and Clamp out Max: clamp final transform value (can be used to limit final transform range of the bone) +- Transform Axis: Which axis this node controls + +### FaceData +This node allows to control arbitary position of Node3d +#### Parameters +- Read From: LiveLinkServer node to read incoming messages from +- Read Shape: Select which shape should be tracked by this node. (Ex. HeadYaw, JawOpen, etc.) +- Target: Set the Node3d that should be controled by this node +- Clamp in Min and Clamp in Max: clamp incoming shape value to the specific range +- Clamp out Min and Clamp out Max: clamp final transform value (can be used to limit final transform range of the bone) +- Transform Axis: Which axis this node controls + +### FaceProperty +This node allows to control any parameter of the node, given it takes number as a value +#### Parameters +- Read From: LiveLinkServer node to read incoming messages from +- Read Shape: Select which shape should be tracked by this node. (Ex. HeadYaw, JawOpen, etc.) +- Target Object: Set any node derived from the Object that should be controled by this node +- Target Property: String path of the controlled property +- Clamp in Min and Clamp in Max: clamp incoming shape value to the specific range +- Clamp out Min and Clamp out Max: clamp final transform value (can be used to limit final transform range of the bone) diff --git a/addons/godot_live_link_face/icon.svg b/addons/godot_live_link_face/icon.svg new file mode 100644 index 0000000..2d7d239 --- /dev/null +++ b/addons/godot_live_link_face/icon.svg @@ -0,0 +1,231 @@ + + +Created with Fabric.js 5.2.4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/addons/godot_live_link_face/icon.svg.import b/addons/godot_live_link_face/icon.svg.import new file mode 100644 index 0000000..614d881 --- /dev/null +++ b/addons/godot_live_link_face/icon.svg.import @@ -0,0 +1,37 @@ +[remap] + +importer="texture" +type="CompressedTexture2D" +uid="uid://d3yh7d8u2sf2b" +path="res://.godot/imported/icon.svg-de16b944993eced7f59f8c80c7744378.ctex" +metadata={ +"vram_texture": false +} + +[deps] + +source_file="res://addons/live_link_face/icon.svg" +dest_files=["res://.godot/imported/icon.svg-de16b944993eced7f59f8c80c7744378.ctex"] + +[params] + +compress/mode=0 +compress/high_quality=false +compress/lossy_quality=0.7 +compress/hdr_compression=1 +compress/normal_map=0 +compress/channel_pack=0 +mipmaps/generate=false +mipmaps/limit=-1 +roughness/mode=0 +roughness/src_normal="" +process/fix_alpha_border=true +process/premult_alpha=true +process/normal_map_invert_y=false +process/hdr_as_srgb=false +process/hdr_clamp_exposure=false +process/size_limit=0 +detect_3d/compress_to=1 +svg/scale=1.0 +editor/scale_with_editor_scale=false +editor/convert_colors_with_editor_theme=false diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..606cf4f --- /dev/null +++ b/flake.lock @@ -0,0 +1,93 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "naersk": { + "inputs": { + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1717067539, + "narHash": "sha256-oIs5EF+6VpHJRvvpVWuqCYJMMVW/6h59aYUv9lABLtY=", + "owner": "nix-community", + "repo": "naersk", + "rev": "fa19d8c135e776dc97f4dcca08656a0eeb28d5c0", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "naersk", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1716948383, + "narHash": "sha256-SzDKxseEcHR5KzPXLwsemyTR/kaM9whxeiJohbL04rs=", + "path": "/nix/store/qgbn0imyridkb9527v6gnv6z3jzzprb9-source", + "rev": "ad57eef4ef0659193044870c731987a6df5cf56b", + "type": "path" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1717112898, + "narHash": "sha256-7R2ZvOnvd9h8fDd65p0JnB7wXfUvreox3xFdYWd1BnY=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "6132b0f6e344ce2fe34fc051b72fb46e34f668e0", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "naersk": "naersk", + "nixpkgs": "nixpkgs_2" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..d0ead69 --- /dev/null +++ b/flake.nix @@ -0,0 +1,47 @@ +{ + inputs = { + flake-utils.url = "github:numtide/flake-utils"; + naersk.url = "github:nix-community/naersk"; + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + }; + + outputs = { self, flake-utils, naersk, nixpkgs }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = (import nixpkgs) { inherit system; }; + naersk' = pkgs.callPackage naersk { }; + libPath = with pkgs; + lib.makeLibraryPath [ + glibc + ]; + + in rec { + # For `nix build` & `nix run`: + packages.default = naersk'.buildPackage { + src = ./.; + pname = "vtub"; + nativeBuildInputs = with pkgs; [ + makeWrapper + pkg-config + openssl + ]; + GIT_HASH = "000000000000000000000000000000"; + postInstall = '' + wrapProgram "$out/bin/${packages.default.pname}" --prefix LD_LIBRARY_PATH : "${libPath}" + ''; + }; + + # For `nix develop`: + devShells.default = pkgs.mkShell { + nativeBuildInputs = with pkgs; [ + godot_4 + ]; + LD_LIBRARY_PATH = libPath; + env = { + VK_LAYER_PATH = "${pkgs.vulkan-validation-layers}/share/vulkan/explicit_layer.d"; + RUST_BACKTRACE = 1; + RUST_LOG = "debug"; + }; + }; + }); +} diff --git a/godot-live-link-face.gdextension b/godot-live-link-face.gdextension new file mode 100644 index 0000000..1daf03f --- /dev/null +++ b/godot-live-link-face.gdextension @@ -0,0 +1,14 @@ +[configuration] +entry_symbol = "gdext_rust_init" +compatibility_minimum = 4.1 +reloadable = true + +[icons] + +LiveLinkServer = "res://addons/godot_live_link_face/icon.svg" + +[libraries] +linux.debug.x86_64 = "res://addons/godot_live_link_face/libgodot_live_link_face.so" +linux.release.x86_64 = "res://addons/godot_live_link_face/libgodot_live_link_face.so" +windows.debug.x86_64 = "res://addons/godot_live_link_face/godot_live_link_face.dll" +windows.release.x86_64 = "res://addons/godot_live_link_face/godot_live_link_face.dll" diff --git a/src/common.rs b/src/common.rs new file mode 100644 index 0000000..caeb240 --- /dev/null +++ b/src/common.rs @@ -0,0 +1,140 @@ +use godot::prelude::*; +use crate::face_data_object::FaceDataObject; + +#[derive(GodotConvert, Var, Export, Debug)] +#[godot(via = GString)] +pub enum ReadBlendShape { + EyeBlinkLeft, + EyeLookDownLeft, + EyeLookInLeft, + EyeLookOutLeft, + EyeLookUpLeft, + EyeSquintLeft, + EyeWideLeft, + EyeBlinkRight, + EyeLookDownRight, + EyeLookInRight, + EyeLookOutRight, + EyeLookUpRight, + EyeSquintRight, + EyeWideRight, + JawForward, + JawLeft, + JawRight, + JawOpen, + MouthClose, + MouthFunnel, + MouthPucker, + MouthLeft, + MouthRight, + MouthSmileLeft, + MouthSmileRight, + MouthFrownLeft, + MouthFrownRight, + MouthDimpleLeft, + MouthDimpleRight, + MouthStretchLeft, + MouthStretchRight, + MouthRollLower, + MouthRollUpper, + MouthShrugLower, + MouthShrugUpper, + MouthPressLeft, + MouthPressRight, + MouthLowerDownLeft, + MouthLowerDownRight, + MouthUpperUpLeft, + MouthUpperUpRight, + BrowDownLeft, + BrowDownRight, + BrowInnerUp, + BrowOuterUpLeft, + BrowOuterUpRight, + CheekPuff, + CheekSquintLeft, + CheekSquintRight, + NoseSneerLeft, + NoseSneerRight, + TongueOut, + HeadYaw, + HeadPitch, + HeadRoll, + LeftEyeYaw, + LeftEyePitch, + LeftEyeRoll, + RightEyeYaw, + RightEyePitch, + RightEyeRoll, +} + + +pub fn map_range_clamped(val: f32, in_min: f32, in_max: f32, out_min: f32, out_max: f32) -> f32 { + (val - in_min) / (in_max - in_min) * (out_max - out_min) + out_min +} + +pub fn map_blend_shape(value: &ReadBlendShape, face_data_object: Gd) -> f32 { + let fdo = face_data_object.bind(); + match value { + ReadBlendShape::EyeBlinkLeft => fdo.get_eye_blink_left(), + ReadBlendShape::EyeLookDownLeft => fdo.get_eye_look_down_left(), + ReadBlendShape::EyeLookInLeft => fdo.get_eye_look_in_left(), + ReadBlendShape::EyeLookOutLeft => fdo.get_eye_look_out_left(), + ReadBlendShape::EyeLookUpLeft => fdo.get_eye_look_up_left(), + ReadBlendShape::EyeSquintLeft => fdo.get_eye_squint_left(), + ReadBlendShape::EyeWideLeft => fdo.get_eye_wide_left(), + ReadBlendShape::EyeBlinkRight => fdo.get_eye_blink_right(), + ReadBlendShape::EyeLookDownRight => fdo.get_eye_look_down_right(), + ReadBlendShape::EyeLookInRight => fdo.get_eye_look_in_right(), + ReadBlendShape::EyeLookOutRight => fdo.get_eye_look_out_right(), + ReadBlendShape::EyeLookUpRight => fdo.get_eye_look_up_right(), + ReadBlendShape::EyeSquintRight => fdo.get_eye_squint_right(), + ReadBlendShape::EyeWideRight => fdo.get_eye_wide_right(), + ReadBlendShape::JawForward => fdo.get_jaw_forward(), + ReadBlendShape::JawLeft => fdo.get_jaw_left(), + ReadBlendShape::JawRight => fdo.get_jaw_right(), + ReadBlendShape::JawOpen => fdo.get_jaw_open(), + ReadBlendShape::MouthClose => fdo.get_mouth_close(), + ReadBlendShape::MouthFunnel => fdo.get_mouth_funnel(), + ReadBlendShape::MouthPucker => fdo.get_mouth_pucker(), + ReadBlendShape::MouthLeft => fdo.get_mouth_left(), + ReadBlendShape::MouthRight => fdo.get_mouth_right(), + ReadBlendShape::MouthSmileLeft => fdo.get_mouth_smile_left(), + ReadBlendShape::MouthSmileRight => fdo.get_mouth_smile_right(), + ReadBlendShape::MouthFrownLeft => fdo.get_mouth_frown_left(), + ReadBlendShape::MouthFrownRight => fdo.get_mouth_frown_right(), + ReadBlendShape::MouthDimpleLeft => fdo.get_mouth_dimple_left(), + ReadBlendShape::MouthDimpleRight => fdo.get_mouth_dimple_right(), + ReadBlendShape::MouthStretchLeft => fdo.get_mouth_stretch_left(), + ReadBlendShape::MouthStretchRight => fdo.get_mouth_stretch_right(), + ReadBlendShape::MouthRollLower => fdo.get_mouth_roll_lower(), + ReadBlendShape::MouthRollUpper => fdo.get_mouth_roll_upper(), + ReadBlendShape::MouthShrugLower => fdo.get_mouth_shrug_lower(), + ReadBlendShape::MouthShrugUpper => fdo.get_mouth_shrug_upper(), + ReadBlendShape::MouthPressLeft => fdo.get_mouth_press_left(), + ReadBlendShape::MouthPressRight => fdo.get_mouth_press_right(), + ReadBlendShape::MouthLowerDownLeft => fdo.get_mouth_lower_down_left(), + ReadBlendShape::MouthLowerDownRight => fdo.get_mouth_lower_down_right(), + ReadBlendShape::MouthUpperUpLeft => fdo.get_mouth_upper_up_left(), + ReadBlendShape::MouthUpperUpRight => fdo.get_mouth_upper_up_right(), + ReadBlendShape::BrowDownLeft => fdo.get_brow_down_left(), + ReadBlendShape::BrowDownRight => fdo.get_brow_down_right(), + ReadBlendShape::BrowInnerUp => fdo.get_brow_inner_up(), + ReadBlendShape::BrowOuterUpLeft => fdo.get_brow_outer_up_left(), + ReadBlendShape::BrowOuterUpRight => fdo.get_brow_outer_up_right(), + ReadBlendShape::CheekPuff => fdo.get_cheek_puff(), + ReadBlendShape::CheekSquintLeft => fdo.get_cheek_squint_left(), + ReadBlendShape::CheekSquintRight => fdo.get_cheek_squint_right(), + ReadBlendShape::NoseSneerLeft => fdo.get_nose_sneer_left(), + ReadBlendShape::NoseSneerRight => fdo.get_nose_sneer_right(), + ReadBlendShape::TongueOut => fdo.get_tongue_out(), + ReadBlendShape::HeadYaw => fdo.get_head_yaw(), + ReadBlendShape::HeadPitch => fdo.get_head_pitch(), + ReadBlendShape::HeadRoll => fdo.get_head_roll(), + ReadBlendShape::LeftEyeYaw => fdo.get_left_eye_yaw(), + ReadBlendShape::LeftEyePitch => fdo.get_left_eye_pitch(), + ReadBlendShape::LeftEyeRoll => fdo.get_left_eye_roll(), + ReadBlendShape::RightEyeYaw => fdo.get_right_eye_yaw(), + ReadBlendShape::RightEyePitch => fdo.get_right_eye_pitch(), + ReadBlendShape::RightEyeRoll => fdo.get_right_eye_roll(), + } +} diff --git a/src/face_bone.rs b/src/face_bone.rs new file mode 100644 index 0000000..719f6ad --- /dev/null +++ b/src/face_bone.rs @@ -0,0 +1,109 @@ +use std::borrow::BorrowMut; + +use godot::{engine::Skeleton3D, prelude::*}; + +use crate::{ + common::{map_blend_shape, map_range_clamped, ReadBlendShape}, + face_data_object::FaceDataObject, + face_transform_rotation::TransformAxis, + live_link_server::LiveLinkServer, +}; + +#[derive(GodotClass)] +#[class(base=Node)] +pub struct FaceBone { + #[export] + read_from: Option>, + #[export] + read_shape: ReadBlendShape, + + #[export] + target_skeleton: Option>, + #[export] + target_bone: GString, + + #[export] + clamp_in_min: f32, + #[export] + clamp_in_max: f32, + #[export] + clamp_out_min: f32, + #[export] + clamp_out_max: f32, + + #[export] + transform_axis: TransformAxis, + + pre_value: f32, + + base: Base, +} + +#[godot_api] +impl FaceBone { + #[func] + fn on_face_data(&mut self, changed: bool, face_data: Option>) { + if changed { + let face_data = face_data.unwrap(); + let shape_val = map_blend_shape(&self.read_shape, face_data); + self.pre_value = shape_val; + } + } +} + +#[godot_api] +impl INode for FaceBone { + fn init(base: Base) -> Self { + Self { + read_from: None, + read_shape: ReadBlendShape::EyeBlinkLeft, + target_skeleton: None, + target_bone: GString::default(), + clamp_in_min: 0.0, + clamp_in_max: 1.0, + clamp_out_min: 0.0, + clamp_out_max: 1.0, + + transform_axis: TransformAxis::X, + + pre_value: 0.0, + + base, + } + } + + fn ready(&mut self) { + if let Some(mut server) = self.borrow_mut().read_from.take() { + let callable = self.base_mut().callable("on_face_data"); + server.connect("face_data_updated".into(), callable); + } + } + + fn process(&mut self, _delta: f64) { + let skeleton = self.get_target_skeleton(); + if let Some(mut target) = skeleton { + let bone_id = target.find_bone(self.get_target_bone()); + let initial_rotation = target.get_bone_pose_rotation(bone_id); + let mut rot_euler = initial_rotation.to_euler(EulerOrder::YXZ); + let clamped = map_range_clamped( + self.pre_value, + self.clamp_in_min, + self.clamp_in_max, + self.clamp_out_min, + self.clamp_out_max, + ); + match self.transform_axis { + TransformAxis::X => { + rot_euler.x = clamped; + } + TransformAxis::Y => { + rot_euler.y = clamped; + } + TransformAxis::Z => { + rot_euler.z = clamped; + } + } + target.set_bone_pose_rotation(bone_id, Quaternion::from_euler(rot_euler).normalized()); + } + } +} diff --git a/src/face_data_object.rs b/src/face_data_object.rs new file mode 100644 index 0000000..8d3601a --- /dev/null +++ b/src/face_data_object.rs @@ -0,0 +1,358 @@ +use godot::prelude::*; +use live_link_face::{FaceBlendShape, LiveLinkFace}; + +#[derive(GodotClass)] +#[class(base=RefCounted)] +pub struct FaceDataObject { + #[var] + eye_blink_left: f32, + #[var] + eye_look_down_left: f32, + #[var] + eye_look_in_left: f32, + #[var] + eye_look_out_left: f32, + #[var] + eye_look_up_left: f32, + #[var] + eye_squint_left: f32, + #[var] + eye_wide_left: f32, + + #[var] + eye_blink_right: f32, + #[var] + eye_look_down_right: f32, + #[var] + eye_look_in_right: f32, + #[var] + eye_look_out_right: f32, + #[var] + eye_look_up_right: f32, + #[var] + eye_squint_right: f32, + #[var] + eye_wide_right: f32, + + #[var] + jaw_forward: f32, + #[var] + jaw_left: f32, + #[var] + jaw_right: f32, + #[var] + jaw_open: f32, + + #[var] + mouth_close: f32, + #[var] + mouth_funnel: f32, + #[var] + mouth_pucker: f32, + #[var] + mouth_left: f32, + #[var] + mouth_right: f32, + #[var] + mouth_smile_left: f32, + #[var] + mouth_smile_right: f32, + #[var] + mouth_frown_left: f32, + #[var] + mouth_frown_right: f32, + #[var] + mouth_dimple_left: f32, + #[var] + mouth_dimple_right: f32, + #[var] + mouth_stretch_left: f32, + #[var] + mouth_stretch_right: f32, + #[var] + mouth_roll_lower: f32, + #[var] + mouth_roll_upper: f32, + #[var] + mouth_shrug_lower: f32, + #[var] + mouth_shrug_upper: f32, + #[var] + mouth_press_left: f32, + #[var] + mouth_press_right: f32, + #[var] + mouth_lower_down_left: f32, + #[var] + mouth_lower_down_right: f32, + #[var] + mouth_upper_up_left: f32, + #[var] + mouth_upper_up_right: f32, + + #[var] + brow_down_left: f32, + #[var] + brow_down_right: f32, + #[var] + brow_inner_up: f32, + #[var] + brow_outer_up_left: f32, + #[var] + brow_outer_up_right: f32, + + #[var] + cheek_puff: f32, + #[var] + cheek_squint_left: f32, + #[var] + cheek_squint_right: f32, + + #[var] + nose_sneer_left: f32, + #[var] + nose_sneer_right: f32, + + #[var] + tongue_out: f32, + + #[var] + head_yaw: f32, + #[var] + head_pitch: f32, + #[var] + head_roll: f32, + + #[var] + left_eye_yaw: f32, + #[var] + left_eye_pitch: f32, + #[var] + left_eye_roll: f32, + + #[var] + right_eye_yaw: f32, + #[var] + right_eye_pitch: f32, + #[var] + right_eye_roll: f32, + + base: Base, +} + +pub fn read_face_data(face_data_object: &mut Gd, face_data: &LiveLinkFace) { + let mut face_data_object = face_data_object.bind_mut(); + + face_data_object.eye_blink_left = face_data.get_blendshape(FaceBlendShape::EyeBlinkLeft); + + face_data_object.eye_look_down_left = face_data.get_blendshape(FaceBlendShape::EyeLookDownLeft); + + face_data_object.eye_look_in_left = face_data.get_blendshape(FaceBlendShape::EyeLookInLeft); + + face_data_object.eye_look_out_left = face_data.get_blendshape(FaceBlendShape::EyeLookOutLeft); + + face_data_object.eye_look_up_left = face_data.get_blendshape(FaceBlendShape::EyeLookUpLeft); + + face_data_object.eye_squint_left = face_data.get_blendshape(FaceBlendShape::EyeSquintLeft); + + face_data_object.eye_wide_left = face_data.get_blendshape(FaceBlendShape::EyeWideLeft); + + face_data_object.eye_blink_right = face_data.get_blendshape(FaceBlendShape::EyeBlinkRight); + + face_data_object.eye_look_down_right = + face_data.get_blendshape(FaceBlendShape::EyeLookDownRight); + + face_data_object.eye_look_in_right = face_data.get_blendshape(FaceBlendShape::EyeLookInRight); + + face_data_object.eye_look_out_right = face_data.get_blendshape(FaceBlendShape::EyeLookOutRight); + + face_data_object.eye_look_up_right = face_data.get_blendshape(FaceBlendShape::EyeLookUpRight); + + face_data_object.eye_squint_right = face_data.get_blendshape(FaceBlendShape::EyeSquintRight); + + face_data_object.eye_wide_right = face_data.get_blendshape(FaceBlendShape::EyeWideRight); + + face_data_object.jaw_forward = face_data.get_blendshape(FaceBlendShape::JawForward); + + face_data_object.jaw_left = face_data.get_blendshape(FaceBlendShape::JawLeft); + + face_data_object.jaw_right = face_data.get_blendshape(FaceBlendShape::JawRight); + + face_data_object.jaw_open = face_data.get_blendshape(FaceBlendShape::JawOpen); + + face_data_object.mouth_close = face_data.get_blendshape(FaceBlendShape::MouthClose); + + face_data_object.mouth_funnel = face_data.get_blendshape(FaceBlendShape::MouthFunnel); + + face_data_object.mouth_pucker = face_data.get_blendshape(FaceBlendShape::MouthPucker); + + face_data_object.mouth_left = face_data.get_blendshape(FaceBlendShape::MouthLeft); + + face_data_object.mouth_right = face_data.get_blendshape(FaceBlendShape::MouthRight); + + face_data_object.mouth_smile_left = face_data.get_blendshape(FaceBlendShape::MouthSmileLeft); + + face_data_object.mouth_smile_right = face_data.get_blendshape(FaceBlendShape::MouthSmileRight); + + face_data_object.mouth_frown_left = face_data.get_blendshape(FaceBlendShape::MouthFrownLeft); + + face_data_object.mouth_frown_right = face_data.get_blendshape(FaceBlendShape::MouthFrownRight); + + face_data_object.mouth_dimple_left = face_data.get_blendshape(FaceBlendShape::MouthDimpleLeft); + + face_data_object.mouth_dimple_right = + face_data.get_blendshape(FaceBlendShape::MouthDimpleRight); + + face_data_object.mouth_stretch_left = + face_data.get_blendshape(FaceBlendShape::MouthStretchLeft); + + face_data_object.mouth_stretch_right = + face_data.get_blendshape(FaceBlendShape::MouthStretchRight); + + face_data_object.mouth_roll_lower = face_data.get_blendshape(FaceBlendShape::MouthRollLower); + + face_data_object.mouth_roll_upper = face_data.get_blendshape(FaceBlendShape::MouthRollUpper); + + face_data_object.mouth_shrug_lower = face_data.get_blendshape(FaceBlendShape::MouthShrugLower); + + face_data_object.mouth_shrug_upper = face_data.get_blendshape(FaceBlendShape::MouthShrugUpper); + + face_data_object.mouth_press_left = face_data.get_blendshape(FaceBlendShape::MouthPressLeft); + + face_data_object.mouth_press_right = face_data.get_blendshape(FaceBlendShape::MouthPressRight); + + face_data_object.mouth_lower_down_left = + face_data.get_blendshape(FaceBlendShape::MouthLowerDownLeft); + + face_data_object.mouth_lower_down_right = + face_data.get_blendshape(FaceBlendShape::MouthLowerDownRight); + + face_data_object.mouth_upper_up_left = + face_data.get_blendshape(FaceBlendShape::MouthUpperUpLeft); + + face_data_object.mouth_upper_up_right = + face_data.get_blendshape(FaceBlendShape::MouthUpperUpRight); + + face_data_object.brow_down_left = face_data.get_blendshape(FaceBlendShape::BrowDownLeft); + + face_data_object.brow_down_right = face_data.get_blendshape(FaceBlendShape::BrowDownRight); + + face_data_object.brow_inner_up = face_data.get_blendshape(FaceBlendShape::BrowInnerUp); + + face_data_object.brow_outer_up_left = face_data.get_blendshape(FaceBlendShape::BrowOuterUpLeft); + + face_data_object.brow_outer_up_right = + face_data.get_blendshape(FaceBlendShape::BrowOuterUpRight); + + face_data_object.cheek_puff = face_data.get_blendshape(FaceBlendShape::CheekPuff); + + face_data_object.cheek_squint_left = face_data.get_blendshape(FaceBlendShape::CheekSquintLeft); + + face_data_object.cheek_squint_right = + face_data.get_blendshape(FaceBlendShape::CheekSquintRight); + + face_data_object.nose_sneer_left = face_data.get_blendshape(FaceBlendShape::NoseSneerLeft); + + face_data_object.nose_sneer_right = face_data.get_blendshape(FaceBlendShape::NoseSneerRight); + + face_data_object.tongue_out = face_data.get_blendshape(FaceBlendShape::TongueOut); + + face_data_object.head_yaw = face_data.get_blendshape(FaceBlendShape::HeadYaw); + + face_data_object.head_pitch = face_data.get_blendshape(FaceBlendShape::HeadPitch); + + face_data_object.head_roll = face_data.get_blendshape(FaceBlendShape::HeadRoll); + + face_data_object.left_eye_yaw = face_data.get_blendshape(FaceBlendShape::LeftEyeYaw); + + face_data_object.left_eye_pitch = face_data.get_blendshape(FaceBlendShape::LeftEyePitch); + + face_data_object.left_eye_roll = face_data.get_blendshape(FaceBlendShape::LeftEyeRoll); + + face_data_object.right_eye_yaw = face_data.get_blendshape(FaceBlendShape::RightEyeYaw); + + face_data_object.right_eye_pitch = face_data.get_blendshape(FaceBlendShape::RightEyePitch); + + face_data_object.right_eye_roll = face_data.get_blendshape(FaceBlendShape::RightEyeRoll); +} + +#[godot_api] +impl IRefCounted for FaceDataObject { + fn init(base: Base) -> Self { + Self { + eye_blink_left: 0.0, + eye_look_down_left: 0.0, + eye_look_in_left: 0.0, + eye_look_out_left: 0.0, + eye_look_up_left: 0.0, + eye_squint_left: 0.0, + eye_wide_left: 0.0, + + eye_blink_right: 0.0, + eye_look_down_right: 0.0, + eye_look_in_right: 0.0, + eye_look_out_right: 0.0, + eye_look_up_right: 0.0, + eye_squint_right: 0.0, + eye_wide_right: 0.0, + + jaw_forward: 0.0, + jaw_left: 0.0, + jaw_right: 0.0, + jaw_open: 0.0, + + mouth_close: 0.0, + mouth_funnel: 0.0, + mouth_pucker: 0.0, + mouth_left: 0.0, + mouth_right: 0.0, + mouth_smile_left: 0.0, + mouth_smile_right: 0.0, + mouth_frown_left: 0.0, + mouth_frown_right: 0.0, + mouth_dimple_left: 0.0, + mouth_dimple_right: 0.0, + mouth_stretch_left: 0.0, + mouth_stretch_right: 0.0, + mouth_roll_lower: 0.0, + mouth_roll_upper: 0.0, + mouth_shrug_lower: 0.0, + mouth_shrug_upper: 0.0, + mouth_press_left: 0.0, + mouth_press_right: 0.0, + mouth_lower_down_left: 0.0, + mouth_lower_down_right: 0.0, + mouth_upper_up_left: 0.0, + mouth_upper_up_right: 0.0, + + brow_down_left: 0.0, + brow_down_right: 0.0, + brow_inner_up: 0.0, + brow_outer_up_left: 0.0, + brow_outer_up_right: 0.0, + + cheek_puff: 0.0, + cheek_squint_left: 0.0, + cheek_squint_right: 0.0, + + nose_sneer_left: 0.0, + nose_sneer_right: 0.0, + + tongue_out: 0.0, + + head_yaw: 0.0, + head_pitch: 0.0, + head_roll: 0.0, + + left_eye_yaw: 0.0, + left_eye_pitch: 0.0, + left_eye_roll: 0.0, + + right_eye_yaw: 0.0, + right_eye_pitch: 0.0, + right_eye_roll: 0.0, + + base, + } + } +} diff --git a/src/face_property.rs b/src/face_property.rs new file mode 100644 index 0000000..de1df50 --- /dev/null +++ b/src/face_property.rs @@ -0,0 +1,91 @@ +use std::borrow::BorrowMut; + +use godot::prelude::*; + +use crate::{ + common::{map_blend_shape, map_range_clamped, ReadBlendShape}, + face_data_object::FaceDataObject, + live_link_server::LiveLinkServer, +}; + +#[derive(GodotClass)] +#[class(base=Node)] +pub struct FaceProperty { + #[export] + read_from: Option>, + #[export] + read_shape: ReadBlendShape, + + #[export] + target_object: Option>, + #[export] + target_property: StringName, + + #[export] + clamp_in_min: f32, + #[export] + clamp_in_max: f32, + #[export] + clamp_out_min: f32, + #[export] + clamp_out_max: f32, + + pre_value: f32, + + base: Base, +} + +#[godot_api] +impl FaceProperty { + #[func] + fn on_face_data(&mut self, changed: bool, face_data: Option>) { + if changed { + let face_data = face_data.unwrap(); + let shape_val = map_blend_shape(&self.read_shape, face_data); + self.pre_value = shape_val; + } + } +} + +#[godot_api] +impl INode for FaceProperty { + fn init(base: Base) -> Self { + Self { + read_from: None, + read_shape: ReadBlendShape::EyeBlinkLeft, + target_object: None, + target_property: StringName::default(), + clamp_in_min: 0.0, + clamp_in_max: 1.0, + clamp_out_min: 0.0, + clamp_out_max: 1.0, + + pre_value: 0.0, + + base, + } + } + + fn ready(&mut self) { + if let Some(mut server) = self.borrow_mut().read_from.take() { + let callable = self.base_mut().callable("on_face_data"); + server.connect("face_data_updated".into(), callable); + } + } + + fn process(&mut self, _delta: f64) { + let object = self.get_target_object(); + let property = self.get_target_property(); + + if let Some(mut target) = object { + let value = map_range_clamped( + self.pre_value, + self.clamp_in_min, + self.clamp_in_max, + self.clamp_out_min, + self.clamp_out_max, + ); + target.set(property.clone(), Variant::from(value)); + } + } +} diff --git a/src/face_transform_rotation.rs b/src/face_transform_rotation.rs new file mode 100644 index 0000000..75b33ef --- /dev/null +++ b/src/face_transform_rotation.rs @@ -0,0 +1,112 @@ +use std::borrow::BorrowMut; + +use godot::prelude::*; + +use crate::{ + common::{map_blend_shape, map_range_clamped, ReadBlendShape}, + face_data_object::FaceDataObject, + live_link_server::LiveLinkServer, +}; + +#[derive(GodotConvert, Var, Export, Debug)] +#[godot(via = GString)] +pub enum TransformAxis { + X, + Y, + Z, +} + +#[derive(GodotClass)] +#[class(base=Node)] +pub struct FaceData { + #[export] + read_from: Option>, + #[export] + read_shape: ReadBlendShape, + + #[export] + target: Option>, + + #[export] + clamp_in_min: f32, + #[export] + clamp_in_max: f32, + #[export] + clamp_out_min: f32, + #[export] + clamp_out_max: f32, + + #[export] + transform_axis: TransformAxis, + + pre_value: f32, + + base: Base, +} + +#[godot_api] +impl FaceData { + #[func] + fn on_face_data(&mut self, changed: bool, face_data: Option>) { + if changed { + let face_data = face_data.unwrap(); + let shape_val = map_blend_shape(&self.read_shape, face_data); + self.pre_value = shape_val; + } + } +} + +#[godot_api] +impl INode for FaceData { + fn init(base: Base) -> Self { + Self { + read_from: None, + read_shape: ReadBlendShape::EyeBlinkLeft, + target: None, + clamp_in_min: 0.0, + clamp_in_max: 1.0, + clamp_out_min: 0.0, + clamp_out_max: 1.0, + + transform_axis: TransformAxis::X, + + pre_value: 0.0, + + base, + } + } + + fn ready(&mut self) { + if let Some(mut server) = self.borrow_mut().read_from.take() { + let callable = self.base_mut().callable("on_face_data"); + server.connect("face_data_updated".into(), callable); + } + } + + fn process(&mut self, _delta: f64) { + if let Some(target) = &mut self.target { + let mut rotation = target.get_rotation_degrees(); + let clamped = map_range_clamped( + self.pre_value, + self.clamp_in_min, + self.clamp_in_max, + self.clamp_out_min, + self.clamp_out_max, + ); + + match self.transform_axis { + TransformAxis::X => { + rotation.x = clamped; + } + TransformAxis::Y => { + rotation.y = clamped; + } + TransformAxis::Z => { + rotation.z = clamped; + } + } + + target.set_rotation_degrees(rotation); + } + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..282962d --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,14 @@ +// use godot::engine::global::Error; +use godot::prelude::*; + +struct LiveLinkExtension; + +#[gdextension] +unsafe impl ExtensionLibrary for LiveLinkExtension {} + +pub mod common; +pub mod live_link_server; +pub mod face_transform_rotation; +pub mod face_data_object; +pub mod face_bone; +pub mod face_property; diff --git a/src/live_link_server.rs b/src/live_link_server.rs new file mode 100644 index 0000000..45deb25 --- /dev/null +++ b/src/live_link_server.rs @@ -0,0 +1,91 @@ +use std::borrow::BorrowMut; + +use godot::engine::global::Error; +use godot::engine::{Node, UdpServer}; +use godot::obj::WithBaseField; +use godot::prelude::*; +use live_link_face::LiveLinkFace; + +use crate::face_data_object::{read_face_data, FaceDataObject}; + +#[derive(GodotClass)] +#[class(base=Node)] +pub struct LiveLinkServer { + #[export] + listen_ip: GString, + #[export] + listen_port: u16, + udp_server: Gd, + + pub face_data: Option, + + base: Base, +} + +#[godot_api] +impl LiveLinkServer { + #[signal] + fn face_data_updated(changed: bool, fd: Option>); +} + +#[godot_api] +impl INode for LiveLinkServer { + fn init(base: Base) -> Self { + let server = UdpServer::new_gd(); + + Self { + listen_ip: "100.64.0.2".into(), + listen_port: 3011, + udp_server: server, + face_data: None, + base, + } + } + + fn ready(&mut self) { + let err = self + .udp_server + .listen_ex(self.listen_port) + .bind_address(self.listen_ip.clone()) + .done(); + + match err { + Error::OK => {} + _ => godot_error!("UdpServer listen error"), + }; + } + + fn enter_tree(&mut self) {} + + fn process(&mut self, _delta: f64) { + self.udp_server.poll(); + + if self.udp_server.is_connection_available() { + if let Some(mut peer) = self.udp_server.take_connection() { + let packet = peer.get_packet(); + + let (has_data, face_data) = LiveLinkFace::decode(packet.as_slice()); + if has_data { + self.borrow_mut().face_data = Some(face_data); + let mut face_data_object = FaceDataObject::new_gd(); + read_face_data( + &mut face_data_object, + &self.borrow_mut().face_data.take().unwrap(), + ); + let fdo_variant = Variant::from(Some(face_data_object)); + let changed = Variant::from(has_data); + self.base_mut() + .emit_signal("face_data_updated".into(), &[changed, fdo_variant]); + } else { + self.borrow_mut().face_data = None; + let changed = Variant::from(has_data); + let fdo_variant = Variant::from::>>(None); + self.base_mut() + .emit_signal("face_data_updated".into(), &[changed, fdo_variant]); + } + } + } + } + + fn exit_tree(&mut self) {} +}