diff --git a/UnityProject/Assets/Plugins/Microsoft.Azure.Kinect.Sensor.dll b/UnityProject/Assets/Plugins/Microsoft.Azure.Kinect.Sensor.dll new file mode 100644 index 0000000..0a43188 Binary files /dev/null and b/UnityProject/Assets/Plugins/Microsoft.Azure.Kinect.Sensor.dll differ diff --git a/UnityProject/Assets/Plugins/Microsoft.Azure.Kinect.Sensor.dll.meta b/UnityProject/Assets/Plugins/Microsoft.Azure.Kinect.Sensor.dll.meta new file mode 100644 index 0000000..779d1fb --- /dev/null +++ b/UnityProject/Assets/Plugins/Microsoft.Azure.Kinect.Sensor.dll.meta @@ -0,0 +1,81 @@ +fileFormatVersion: 2 +guid: 77ab96b9ed3028048bdece86f38c92c9 +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 0 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + '': Any + second: + enabled: 0 + settings: + Exclude Editor: 0 + Exclude Linux64: 0 + Exclude OSXUniversal: 0 + Exclude Win: 0 + Exclude Win64: 0 + - first: + Any: + second: + enabled: 1 + settings: {} + - first: + Editor: Editor + second: + enabled: 1 + settings: + CPU: AnyCPU + DefaultValueInitialized: true + OS: AnyOS + - first: + Facebook: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Facebook: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Linux64 + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: OSXUniversal + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: Win + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: Win64 + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Windows Store Apps: WindowsStoreApps + second: + enabled: 0 + settings: + CPU: AnyCPU + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/RenderTextures.meta b/UnityProject/Assets/RenderTextures.meta new file mode 100644 index 0000000..448a382 --- /dev/null +++ b/UnityProject/Assets/RenderTextures.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: a727511980477174bb68636438f49427 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/RenderTextures/Color.renderTexture b/UnityProject/Assets/RenderTextures/Color.renderTexture new file mode 100644 index 0000000..6e146e5 --- /dev/null +++ b/UnityProject/Assets/RenderTextures/Color.renderTexture @@ -0,0 +1,37 @@ +%YAML 1.1 +%TAG !u! tag:unity3d.com,2011: +--- !u!84 &8400000 +RenderTexture: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_Name: Color + m_ImageContentsHash: + serializedVersion: 2 + Hash: 00000000000000000000000000000000 + m_ForcedFallbackFormat: 4 + m_DownscaleFallback: 0 + serializedVersion: 3 + m_Width: 2048 + m_Height: 1536 + m_AntiAliasing: 1 + m_MipCount: -1 + m_DepthFormat: 0 + m_ColorFormat: 8 + m_MipMap: 0 + m_GenerateMips: 1 + m_SRGB: 0 + m_UseDynamicScale: 0 + m_BindMS: 0 + m_EnableCompatibleFormat: 1 + m_TextureSettings: + serializedVersion: 2 + m_FilterMode: 0 + m_Aniso: 0 + m_MipBias: 0 + m_WrapU: 1 + m_WrapV: 1 + m_WrapW: 1 + m_Dimension: 2 + m_VolumeDepth: 1 diff --git a/UnityProject/Assets/RenderTextures/Color.renderTexture.meta b/UnityProject/Assets/RenderTextures/Color.renderTexture.meta new file mode 100644 index 0000000..f348a2c --- /dev/null +++ b/UnityProject/Assets/RenderTextures/Color.renderTexture.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 046aa26a9af8e164eb0b20672983a11f +NativeFormatImporter: + externalObjects: {} + mainObjectFileID: 0 + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/RenderTextures/Position.renderTexture b/UnityProject/Assets/RenderTextures/Position.renderTexture new file mode 100644 index 0000000..cd51da5 --- /dev/null +++ b/UnityProject/Assets/RenderTextures/Position.renderTexture @@ -0,0 +1,37 @@ +%YAML 1.1 +%TAG !u! tag:unity3d.com,2011: +--- !u!84 &8400000 +RenderTexture: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_Name: Position + m_ImageContentsHash: + serializedVersion: 2 + Hash: 00000000000000000000000000000000 + m_ForcedFallbackFormat: 4 + m_DownscaleFallback: 0 + serializedVersion: 3 + m_Width: 2048 + m_Height: 1536 + m_AntiAliasing: 1 + m_MipCount: -1 + m_DepthFormat: 0 + m_ColorFormat: 48 + m_MipMap: 0 + m_GenerateMips: 1 + m_SRGB: 0 + m_UseDynamicScale: 0 + m_BindMS: 0 + m_EnableCompatibleFormat: 1 + m_TextureSettings: + serializedVersion: 2 + m_FilterMode: 0 + m_Aniso: 0 + m_MipBias: 0 + m_WrapU: 1 + m_WrapV: 1 + m_WrapW: 1 + m_Dimension: 2 + m_VolumeDepth: 1 diff --git a/UnityProject/Assets/RenderTextures/Position.renderTexture.meta b/UnityProject/Assets/RenderTextures/Position.renderTexture.meta new file mode 100644 index 0000000..9bba963 --- /dev/null +++ b/UnityProject/Assets/RenderTextures/Position.renderTexture.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: f235609b825377142ab45895e91b6839 +NativeFormatImporter: + externalObjects: {} + mainObjectFileID: 0 + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/Scenes/Examples/Azure/AzureDemo.unity.meta b/UnityProject/Assets/Scenes/Examples/Azure/AzureDemo.unity.meta index c2c33ce..2fea0e8 100644 --- a/UnityProject/Assets/Scenes/Examples/Azure/AzureDemo.unity.meta +++ b/UnityProject/Assets/Scenes/Examples/Azure/AzureDemo.unity.meta @@ -1,5 +1,10 @@ fileFormatVersion: 2 +<<<<<<< HEAD:UnityProject/Assets/Scenes/Examples/Azure/AzureDemo.unity.meta guid: d2b605d5e6f33cb43a6b49852af3bc24 +======= +guid: df11262fc1b053a488b5cfbf724e4cda +folderAsset: yes +>>>>>>> refs/heads/master:UnityProject/Assets/Scenes/Examples/Azure.meta DefaultImporter: externalObjects: {} userData: diff --git a/UnityProject/Assets/Settings.meta b/UnityProject/Assets/Settings.meta new file mode 100644 index 0000000..3746190 --- /dev/null +++ b/UnityProject/Assets/Settings.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: c88789b858a250f4fb98c3d6eab7a086 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/Settings/DeviceSetting.asset b/UnityProject/Assets/Settings/DeviceSetting.asset new file mode 100644 index 0000000..dd56c6f --- /dev/null +++ b/UnityProject/Assets/Settings/DeviceSetting.asset @@ -0,0 +1,26 @@ +%YAML 1.1 +%TAG !u! tag:unity3d.com,2011: +--- !u!114 &11400000 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 0} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: daa4216d09a83dc479a7ee221b2a5bf9, type: 3} + m_Name: Akvfx Settings + m_EditorClassIdentifier: + _autoExposure: 1 + _exposure: 0.5 + _autoWhiteBalance: 1 + _whiteBalance: 0.5 + _brightness: 0.5 + _contrast: 0.5 + _saturation: 0.5 + _sharpness: 0.5 + _gain: 1 + _enableBlc: 0 + _powerIs60Hz: 1 + _maxDepth: 1 diff --git a/UnityProject/Assets/Settings/DeviceSetting.asset.meta b/UnityProject/Assets/Settings/DeviceSetting.asset.meta new file mode 100644 index 0000000..d48626d --- /dev/null +++ b/UnityProject/Assets/Settings/DeviceSetting.asset.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: f8122729f4736dd4ab99617ee19feec3 +NativeFormatImporter: + externalObjects: {} + mainObjectFileID: 11400000 + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/ThridParty/Keijiro.meta b/UnityProject/Assets/ThridParty/Keijiro.meta new file mode 100644 index 0000000..c61c543 --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 42ba717c0afb14b4294df4939f72246d +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/ThridParty/Keijiro/DeviceSettings.cs b/UnityProject/Assets/ThridParty/Keijiro/DeviceSettings.cs new file mode 100644 index 0000000..42bd903 --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/DeviceSettings.cs @@ -0,0 +1,136 @@ +using UnityEngine; + +namespace Akvfx +{ + public sealed class DeviceSettings : ScriptableObject + { + #region Editable fields + + [SerializeField] bool _autoExposure = true; + [SerializeField, Range(0, 1)] float _exposure = 0.5f; + + [SerializeField] bool _autoWhiteBalance = true; + [SerializeField, Range(0, 1)] float _whiteBalance = 0.5f; + + [SerializeField, Range(0, 1)] float _brightness = 0.5f; + [SerializeField, Range(0, 1)] float _contrast = 0.5f; + [SerializeField, Range(0, 1)] float _saturation = 0.5f; + [SerializeField, Range(0, 1)] float _sharpness = 0.5f; + [SerializeField, Range(0, 1)] float _gain = 1; + + [SerializeField] bool _enableBlc = false; + [SerializeField] bool _powerIs60Hz = true; + + [SerializeField, Range(0, 6.6f)] float _maxDepth = 1; + + #endregion + + #region Public accessors + + public bool autoExposure { + get { return _autoExposure; } + set { _autoExposure = value; } + } + + public float exposure { + get { return _exposure; } + set { _exposure = value; } + } + + public bool autoWhiteBalance { + get { return _autoWhiteBalance; } + set { _autoWhiteBalance = value; } + } + + public float whiteBalance { + get { return _whiteBalance; } + set { _whiteBalance = value; } + } + + public float brightness { + get { return _brightness; } + set { _brightness = value; } + } + + public float contrast { + get { return _contrast; } + set { _contrast = value; } + } + + public float saturation { + get { return _saturation; } + set { _saturation = value; } + } + + public float sharpness { + get { return _sharpness; } + set { _sharpness = value; } + } + + public float gain { + get { return _gain; } + set { _gain = value; } + } + + public bool enableBlc { + get { return _enableBlc; } + set { _enableBlc = value; } + } + + public bool powerIs60Hz { + get { return _powerIs60Hz; } + set { _powerIs60Hz = value; } + } + + public float maxDepth { + get { return _maxDepth; } + set { _maxDepth = value; } + } + + #endregion + + #region Internal properties + + internal int ExposureDeviceValue { get { + if (_autoExposure) return -1; + var exp = Mathf.Pow(_exposure, 8); + return (int)Mathf.Lerp(488.0f, 1000000.0f, exp); + } } + + internal int WhiteBalanceDeviceValue { get { + if (_autoWhiteBalance) return -1; + var x = (int)Mathf.Lerp(2500, 10000, _whiteBalance); + return x - x % 10; // Should be divisible by 10. + } } + + internal int BrightnessDeviceValue { get { + return (int)Mathf.Lerp(0, 255, _brightness); + } } + + internal int ContrastDeviceValue { get { + return (int)Mathf.Lerp(0, 10, _contrast); + } } + + internal int SaturationDeviceValue { get { + return (int)Mathf.Lerp(0, 63, _saturation); + } } + + internal int SharpnessDeviceValue { get { + return (int)Mathf.Lerp(0, 4, _sharpness); + } } + + internal int GainDeviceValue { get { + return (int)Mathf.Lerp(0, 255, _gain); + } } + + internal int BlcDeviceValue { get { + return _enableBlc ? 1 : 0; + } } + + internal int PowerFreqDeviceValue { get { + return _powerIs60Hz ? 2 : 1; + } } + + #endregion + } +} diff --git a/UnityProject/Assets/ThridParty/Keijiro/DeviceSettings.cs.meta b/UnityProject/Assets/ThridParty/Keijiro/DeviceSettings.cs.meta new file mode 100644 index 0000000..0a32ffe --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/DeviceSettings.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: daa4216d09a83dc479a7ee221b2a5bf9 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/ThridParty/Keijiro/Editor.meta b/UnityProject/Assets/ThridParty/Keijiro/Editor.meta new file mode 100644 index 0000000..528ea96 --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/Editor.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 9aff79d67109d4f4cb7dd61f35f6e452 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/ThridParty/Keijiro/Editor/DeviceSettingsEditor.cs b/UnityProject/Assets/ThridParty/Keijiro/Editor/DeviceSettingsEditor.cs new file mode 100644 index 0000000..06bc0f2 --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/Editor/DeviceSettingsEditor.cs @@ -0,0 +1,104 @@ +using UnityEngine; +using UnityEditor; + +namespace Akvfx +{ + [CanEditMultipleObjects] + [CustomEditor(typeof(DeviceSettings))] + sealed class DeviceSettingsEditor : Editor + { + SerializedProperty _autoExposure; + SerializedProperty _exposure; + + SerializedProperty _autoWhiteBalance; + SerializedProperty _whiteBalance; + + SerializedProperty _brightness; + SerializedProperty _contrast; + SerializedProperty _saturation; + SerializedProperty _sharpness; + SerializedProperty _gain; + + SerializedProperty _enableBlc; + SerializedProperty _powerIs60Hz; + + SerializedProperty _maxDepth; + + static readonly ( + GUIContent enableBlc, + GUIContent powerIs60Hz + ) _labels = ( + new GUIContent("Enable BLC"), + new GUIContent("Power is 60Hz") + ); + + void OnEnable() + { + _autoExposure = serializedObject.FindProperty("_autoExposure"); + _exposure = serializedObject.FindProperty("_exposure"); + + _autoWhiteBalance = serializedObject.FindProperty("_autoWhiteBalance"); + _whiteBalance = serializedObject.FindProperty("_whiteBalance"); + + _brightness = serializedObject.FindProperty("_brightness"); + _contrast = serializedObject.FindProperty("_contrast"); + _saturation = serializedObject.FindProperty("_saturation"); + _sharpness = serializedObject.FindProperty("_sharpness"); + _gain = serializedObject.FindProperty("_gain"); + + _enableBlc = serializedObject.FindProperty("_enableBlc"); + _powerIs60Hz = serializedObject.FindProperty("_powerIs60Hz"); + + _maxDepth = serializedObject.FindProperty("_maxDepth"); + } + + public override void OnInspectorGUI() + { + serializedObject.Update(); + + EditorGUILayout.PropertyField(_autoExposure); + if (_autoExposure.hasMultipleDifferentValues || + !_autoExposure.boolValue) + { + EditorGUI.indentLevel++; + EditorGUILayout.PropertyField(_exposure); + EditorGUI.indentLevel--; + } + + EditorGUILayout.PropertyField(_autoWhiteBalance); + if (_autoWhiteBalance.hasMultipleDifferentValues || + !_autoWhiteBalance.boolValue) + { + EditorGUI.indentLevel++; + EditorGUILayout.PropertyField(_whiteBalance); + EditorGUI.indentLevel--; + } + + EditorGUILayout.PropertyField(_brightness); + EditorGUILayout.PropertyField(_contrast); + EditorGUILayout.PropertyField(_saturation); + EditorGUILayout.PropertyField(_sharpness); + EditorGUILayout.PropertyField(_gain); + + EditorGUILayout.PropertyField(_enableBlc, _labels.enableBlc); + EditorGUILayout.PropertyField(_powerIs60Hz, _labels.powerIs60Hz); + + EditorGUILayout.PropertyField(_maxDepth); + + serializedObject.ApplyModifiedProperties(); + } + + [MenuItem("Assets/Create/Akvfx/Device Settings")] + public static void CreateDeviceSettings() + { + var asset = ScriptableObject.CreateInstance(); + + AssetDatabase.CreateAsset(asset, "Assets/Akvfx Settings.asset"); + AssetDatabase.SaveAssets(); + + EditorUtility.FocusProjectWindow(); + + Selection.activeObject = asset; + } + } +} diff --git a/UnityProject/Assets/ThridParty/Keijiro/Editor/DeviceSettingsEditor.cs.meta b/UnityProject/Assets/ThridParty/Keijiro/Editor/DeviceSettingsEditor.cs.meta new file mode 100644 index 0000000..0048c39 --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/Editor/DeviceSettingsEditor.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 39b37077152d5024ab6ca873d5de277f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/ThridParty/Keijiro/Internal.meta b/UnityProject/Assets/ThridParty/Keijiro/Internal.meta new file mode 100644 index 0000000..94ea91b --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/Internal.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 84f2346cb4622004499b20cda8c2bee0 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/ThridParty/Keijiro/Internal/DeviceSettingController.cs b/UnityProject/Assets/ThridParty/Keijiro/Internal/DeviceSettingController.cs new file mode 100644 index 0000000..68d2ba7 --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/Internal/DeviceSettingController.cs @@ -0,0 +1,110 @@ +using Microsoft.Azure.Kinect.Sensor; + +namespace Akvfx +{ + sealed class DeviceSettingController + { + #region Public methods + + public DeviceSettingController(Device device, DeviceSettings initial) + { + ApplyInternal(device, initial, true); + } + + public void ApplySettings(Device device, DeviceSettings settings) + { + ApplyInternal(device, settings, false); + } + + #endregion + + #region Cached control values + + int _exposure; + int _whiteBalance; + int _brightness; + int _contrast; + int _saturation; + int _sharpness; + int _gain; + int _blc; + int _powerFreq; + + #endregion + + #region Private methods + + int ApplyControl( + Device device, ColorControlCommand command, + int newValue, int prevValue, bool forceApply + ) + { + // If nothing was changed, simply return the previous value. + if (!forceApply && newValue == prevValue) return prevValue; + + // Apply the new value to the control. + device.SetColorControl( + command, + newValue < 0 ? ColorControlMode.Auto : ColorControlMode.Manual, + newValue < 0 ? 0 : newValue + ); + return newValue; + } + + void ApplyInternal(Device device, DeviceSettings settings, bool forceApply) + { + _exposure = ApplyControl( + device, ColorControlCommand.ExposureTimeAbsolute, + settings.ExposureDeviceValue, _exposure, forceApply + ); + + _whiteBalance = ApplyControl( + device, ColorControlCommand.Whitebalance, + settings.WhiteBalanceDeviceValue, _whiteBalance, forceApply + ); + + _brightness = ApplyControl( + device, ColorControlCommand.Brightness, + settings.BrightnessDeviceValue, _brightness, forceApply + ); + + _contrast = ApplyControl( + device, ColorControlCommand.Contrast, + settings.ContrastDeviceValue, _contrast, forceApply + ); + + _saturation = ApplyControl( + device, ColorControlCommand.Saturation, + settings.SaturationDeviceValue, _saturation, forceApply + ); + + _sharpness = ApplyControl( + device, ColorControlCommand.Sharpness, + settings.SharpnessDeviceValue, _sharpness, forceApply + ); + + // This is not documented, but the gain parameter can't update + // while the auto exposure is enabled. To delay updates, we do a + // bit tricky thing here. + if (_exposure < 0 || forceApply) + _gain = -1; + else + _gain = ApplyControl( + device, ColorControlCommand.Gain, + settings.GainDeviceValue, _gain, false + ); + + _blc = ApplyControl( + device, ColorControlCommand.BacklightCompensation, + settings.BlcDeviceValue, _blc, forceApply + ); + + _powerFreq = ApplyControl( + device, ColorControlCommand.PowerlineFrequency, + settings.PowerFreqDeviceValue, _powerFreq, forceApply + ); + } + + #endregion + } +} diff --git a/UnityProject/Assets/ThridParty/Keijiro/Internal/DeviceSettingController.cs.meta b/UnityProject/Assets/ThridParty/Keijiro/Internal/DeviceSettingController.cs.meta new file mode 100644 index 0000000..fbeec1d --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/Internal/DeviceSettingController.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 492dfb729c0136d48b2917cc16811e38 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/ThridParty/Keijiro/Internal/GraphicsExtensions.cs b/UnityProject/Assets/ThridParty/Keijiro/Internal/GraphicsExtensions.cs new file mode 100644 index 0000000..9dfd10a --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/Internal/GraphicsExtensions.cs @@ -0,0 +1,75 @@ +// +// This file contains some utility extensions for Unity graphics classes. +// These extensions are mainly for adding Span support to storage classes. +// + +using UnityEngine; +using System; +using System.Reflection; + +namespace Akvfx +{ + static class GraphicsExtensions + { + // MRT with tow render textures + public static void SetRenderTarget(RenderTexture rt1, RenderTexture rt2) + { + _target2[0] = rt1.colorBuffer; + _target2[1] = rt2.colorBuffer; + Graphics.SetRenderTarget(_target2, rt1.depthBuffer); + } + + static RenderBuffer [] _target2 = new RenderBuffer[2]; + } + + static class Texture2DExtensions + { + // LoadRawTextureData with ReadOnlySpan + public unsafe static void LoadRawTextureData + (this Texture2D texture, ReadOnlySpan data) + { + fixed (byte* pData = &data.GetPinnableReference()) + texture.LoadRawTextureData((IntPtr)pData, data.Length); + } + } + + static class ComputeBufferExtensions + { + // SetData with ReadOnlySpan + public unsafe static void SetData + (this ComputeBuffer buffer, ReadOnlySpan data) + where T : unmanaged + { + fixed (T* pData = &data.GetPinnableReference()) + buffer.SetData((IntPtr)pData, data.Length, sizeof(T)); + } + + // Directly load an unmanaged memory block to a compute buffer via an + // Intptr. This is not a public interface so will be broken one day. + // DO NOT TRY AT HOME. + public static void SetData + (this ComputeBuffer buffer, IntPtr pointer, int count, int stride) + { + if (_method == null) + { + _method = typeof(ComputeBuffer).GetMethod( + "InternalSetNativeData", + BindingFlags.InvokeMethod | + BindingFlags.NonPublic | + BindingFlags.Instance + ); + } + + _args5[0] = pointer; + _args5[1] = 0; // source offset + _args5[2] = 0; // buffer offset + _args5[3] = count; + _args5[4] = stride; + + _method.Invoke(buffer, _args5); + } + + static MethodInfo _method; + static object [] _args5 = new object[5]; + } +} diff --git a/UnityProject/Assets/ThridParty/Keijiro/Internal/GraphicsExtensions.cs.meta b/UnityProject/Assets/ThridParty/Keijiro/Internal/GraphicsExtensions.cs.meta new file mode 100644 index 0000000..a6f8918 --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/Internal/GraphicsExtensions.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 01b4ad67ec4d57f4d958129b1d240d75 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/ThridParty/Keijiro/Internal/K4aExtensions.cs b/UnityProject/Assets/ThridParty/Keijiro/Internal/K4aExtensions.cs new file mode 100644 index 0000000..a11210e --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/Internal/K4aExtensions.cs @@ -0,0 +1,29 @@ +using System.Reflection; + +namespace Akvfx +{ + static class K4aExtensions + { + // Set false to Allocator.SafeCopyNativeBuffers. You can earn a few + // milliseconds by skipping safe-copy. Note that it turns minor bugs + // undebuggable crashes. + public static void DisableSafeCopyNativeBuffers() + { + var allocator = System.Type.GetType( + "Microsoft.Azure.Kinect.Sensor.Allocator,Microsoft.Azure.Kinect.Sensor" + ); + + var singleton = allocator.GetProperty( + "Singleton", + BindingFlags.Public | BindingFlags.Static + ); + + var safeCopyNativeBuffers = allocator.GetProperty( + "SafeCopyNativeBuffers", + BindingFlags.Public | BindingFlags.Instance + ); + + safeCopyNativeBuffers.SetValue(singleton.GetValue(null), false); + } + } +} diff --git a/UnityProject/Assets/ThridParty/Keijiro/Internal/K4aExtensions.cs.meta b/UnityProject/Assets/ThridParty/Keijiro/Internal/K4aExtensions.cs.meta new file mode 100644 index 0000000..fc08c99 --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/Internal/K4aExtensions.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 275db18c032614049a8e1b71387b610a +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/ThridParty/Keijiro/Internal/ThreadedDriver.cs b/UnityProject/Assets/ThridParty/Keijiro/Internal/ThreadedDriver.cs new file mode 100644 index 0000000..2538f7c --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/Internal/ThreadedDriver.cs @@ -0,0 +1,150 @@ +using Microsoft.Azure.Kinect.Sensor; +using System; +using System.Collections.Concurrent; +using System.Threading; +using System.Threading.Tasks; + +namespace Akvfx +{ + sealed class ThreadedDriver : IDisposable + { + #region Public properties and methods + + public ThreadedDriver(DeviceSettings settings) + { + // FIXME: Dangerous. We should do this only on Player. + K4aExtensions.DisableSafeCopyNativeBuffers(); + + _settings = settings; + + _captureThread = new Thread(CaptureThread); + _captureThread.Start(); + } + + public void Dispose() + { + _terminate = true; + _captureThread.Join(); + + TrimQueue(0); + ReleaseLastFrame(); + + GC.SuppressFinalize(this); + } + + public ReadOnlySpan XYTable + { + get { return _xyTable != null ? _xyTable.Data : null; } + } + + public (ReadOnlyMemory color, + ReadOnlyMemory depth) LockLastFrame() + { + // Try retrieving the last frame. + if (_lockedFrame.capture == null) + _queue.TryDequeue(out _lockedFrame); + + // Return null if it failed to retrieve. + if (_lockedFrame.capture == null) return (null, null); + + return (_lockedFrame.capture.Color.Memory, + _lockedFrame.depth.Memory); + } + + public void ReleaseLastFrame() + { + _lockedFrame.capture?.Dispose(); + _lockedFrame.depth?.Dispose(); + _lockedFrame = (null, null); + } + + #endregion + + #region Private objects + + DeviceSettings _settings; + XYTable _xyTable; + + #endregion + + #region Capture queue + + ConcurrentQueue<(Capture capture, Image depth)> + _queue = new ConcurrentQueue<(Capture, Image)>(); + + (Capture capture, Image depth) _lockedFrame; + + // Trim the queue to a specified count. + void TrimQueue(int count) + { + while (_queue.Count > count) + { + (Capture capture, Image depth) temp; + _queue.TryDequeue(out temp); + temp.capture?.Dispose(); + temp.depth?.Dispose(); + } + } + + #endregion + + #region Capture thread + + Thread _captureThread; + bool _terminate; + + void CaptureThread() + { + // If there is no available device, do nothing. + if (Device.GetInstalledCount() == 0) return; + + // Open the default device. + var device = Device.Open(); + + // Start capturing with custom settings. + device.StartCameras( + new DeviceConfiguration { + ColorFormat = ImageFormat.ColorBGRA32, + ColorResolution = ColorResolution.R1536p, // 2048 x 1536 (4:3) + DepthMode = DepthMode.NFOV_Unbinned, // 640x576 + SynchronizedImagesOnly = true + } + ); + + // Construct XY table as a background task. + Task.Run(() => { + _xyTable = new XYTable(device.GetCalibration(), 2048, 1536); + }); + + // Set up the transformation object. + var transformation = new Transformation(device.GetCalibration()); + + // Initially apply the device settings. + var setter = new DeviceSettingController(device, _settings); + + while (!_terminate) + { + // Get a frame capture. + var capture = device.GetCapture(); + + // Transform the depth image to the color perspective. + var depth = transformation.DepthImageToColorCamera(capture); + + // Push the frame to the capture queue. + _queue.Enqueue((capture, depth)); + + // Remove old frames. + TrimQueue(1); + + // Apply changes on the device settings. + setter.ApplySettings(device, _settings); + } + + // Cleaning up. + transformation.Dispose(); + device.Dispose(); + } + + #endregion + } +} diff --git a/UnityProject/Assets/ThridParty/Keijiro/Internal/ThreadedDriver.cs.meta b/UnityProject/Assets/ThridParty/Keijiro/Internal/ThreadedDriver.cs.meta new file mode 100644 index 0000000..a66ef0c --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/Internal/ThreadedDriver.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: b43ac31f3a29da64ba703a0bcf364a95 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/ThridParty/Keijiro/Internal/Unproject.shader b/UnityProject/Assets/ThridParty/Keijiro/Internal/Unproject.shader new file mode 100644 index 0000000..5f6effc --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/Internal/Unproject.shader @@ -0,0 +1,67 @@ +Shader "Hidden/Akvfx/Unproject" +{ + CGINCLUDE + + #include "UnityCG.cginc" + + texture2D _ColorTexture; + texture2D _DepthTexture; + StructuredBuffer _XYTable; + float _MaxDepth; + + void Vertex( + float4 position : POSITION, + out float4 positionOut : SV_Position, + inout float2 texCoord : TEXCOORD0 + ) + { + positionOut = UnityObjectToClipPos(position); + } + + void Fragment( + float4 position : SV_Position, + float2 texCoord : TEXCOORD0, + out float4 colorOut : SV_Target0, + out float4 positionOut : SV_Target1 + ) + { + uint w, h; + _ColorTexture.GetDimensions(w, h); + + // Texture index + uint tx = texCoord.x * w; + uint ty = texCoord.y * h; + + // Color sample + float4 color = _ColorTexture[uint2(tx, ty)]; + + // Depth sample (int16 -> float) + int d0 = _DepthTexture[uint2(tx * 2 + 0, ty)] * 255; + int d1 = _DepthTexture[uint2(tx * 2 + 1, ty)] * 255; + float depth = (float)(d0 + (d1 << 8)) / 1000; + float mask = depth > 0 && depth < _MaxDepth; + float z = lerp(_MaxDepth, depth, mask); + + // XY table lookup + uint xy_i = (tx + ty * w) * 2; + float2 xy = float2(_XYTable[xy_i], -_XYTable[xy_i + 1]); + + // MRT output write + colorOut = float4(color.rgb, mask); + positionOut = float4(xy * z, z, mask); + } + + ENDCG + + SubShader + { + Cull Off ZWrite Off ZTest Always + Pass + { + CGPROGRAM + #pragma vertex Vertex + #pragma fragment Fragment + ENDCG + } + } +} diff --git a/UnityProject/Assets/ThridParty/Keijiro/Internal/Unproject.shader.meta b/UnityProject/Assets/ThridParty/Keijiro/Internal/Unproject.shader.meta new file mode 100644 index 0000000..b2a2cba --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/Internal/Unproject.shader.meta @@ -0,0 +1,9 @@ +fileFormatVersion: 2 +guid: c2bfd9668998b6848ac60594704c344c +ShaderImporter: + externalObjects: {} + defaultTextures: [] + nonModifiableTextures: [] + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/ThridParty/Keijiro/Internal/XYTable.cs b/UnityProject/Assets/ThridParty/Keijiro/Internal/XYTable.cs new file mode 100644 index 0000000..ff31af6 --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/Internal/XYTable.cs @@ -0,0 +1,80 @@ +using Microsoft.Azure.Kinect.Sensor; +using System; +using System.Runtime.InteropServices; +using System.Threading.Tasks; + +namespace Akvfx +{ + // + // Per pixel ray direction table ("XY table") used to unproject depth + // samples to the 3D camera space + // + // This class directly invokes a native method (k4a_calibration_2d_to_3d) + // contained in libk4a to avoid double symbol definition problem between + // System.Numerics and System.Numerics.Vectors. + // + sealed class XYTable + { + // Public property: Table data + public ReadOnlySpan Data { get { return _data; } } + + // Float vvector types + struct Float2 { public float x, y; } + struct Float3 { public float x, y, z; } + + // Data storage + float [] _data; + + // Constructor + public XYTable(Calibration calibration, int width, int height) + { + // Data storage allocation + var table = new float [width * height * 2]; + + // XY table initialization + // These loops could take a significant amount of time (e.g. 1 sec) + // even on a decent desktop PC. Luckily, this can be easily + // parallelized by using Parallel.For, so we did. + Parallel.For(0, height, y => { + Float2 v2; + Float3 v3; + bool isValid; + + v2.y = y; + var offs = width * 2 * y; + + for (var x = 0; x < width; x++) + { + v2.x = x; + + k4a_calibration_2d_to_3d( + ref calibration, + ref v2, 1, + CalibrationDeviceType.Color, + CalibrationDeviceType.Color, + out v3, + out isValid + ); + + table[offs++] = v3.x; + table[offs++] = v3.y; + } + }); + + // Publish the table data. + _data = table; + } + + // k4a_calibration_2d_to_3d native method from libk4a + [DllImport("k4a", CallingConvention = CallingConvention.Cdecl)] + static extern int k4a_calibration_2d_to_3d( + [In] ref Calibration calibration, + ref Float2 source_point2d, + float source_depth, + CalibrationDeviceType source_camera, + CalibrationDeviceType target_camera, + out Float3 target_point3d, + out bool valid + ); + } +} diff --git a/UnityProject/Assets/ThridParty/Keijiro/Internal/XYTable.cs.meta b/UnityProject/Assets/ThridParty/Keijiro/Internal/XYTable.cs.meta new file mode 100644 index 0000000..14b2835 --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/Internal/XYTable.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: efd020f99d8831e4a9f43be37d1de501 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Assets/ThridParty/Keijiro/PointCloudBaker.cs b/UnityProject/Assets/ThridParty/Keijiro/PointCloudBaker.cs new file mode 100644 index 0000000..243f7cb --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/PointCloudBaker.cs @@ -0,0 +1,91 @@ +using UnityEngine; +using GraphicsFormat = UnityEngine.Experimental.Rendering.GraphicsFormat; + +namespace Akvfx +{ + public sealed class PointCloudBaker : MonoBehaviour + { + #region Editable attributes + + [SerializeField] DeviceSettings _deviceSettings = null; + [SerializeField] RenderTexture _colorTexture = null; + [SerializeField] RenderTexture _positionTexture = null; + [SerializeField] Shader _shader = null; + + #endregion + + #region Internal objects + + ThreadedDriver _driver; + Material _material; + ComputeBuffer _xyTable; + (Texture2D color, Texture2D depth) _temporaries; + + #endregion + + #region MonoBehaviour implementation + + void Start() + { + // Start capturing via the threaded driver. + _driver = new ThreadedDriver(_deviceSettings); + + // Temporary objects for convertion shader + _material = new Material(_shader); + _temporaries = ( + new Texture2D(2048, 1536, GraphicsFormat.B8G8R8A8_SRGB, 0), + new Texture2D(2048 * 2, 1536, GraphicsFormat.R8_UNorm, 0) + ); + } + + void OnDestroy() + { + if (_material != null) Destroy(_material); + _xyTable?.Dispose(); + if (_temporaries.color != null) Destroy(_temporaries.color); + if (_temporaries.depth != null) Destroy(_temporaries.depth); + _driver?.Dispose(); + } + + RenderBuffer[] _mrt = new RenderBuffer[2]; + + unsafe void Update() + { + // Try initializing XY table if it's not ready. + if (_xyTable == null) + { + var data = _driver.XYTable; + if (data.IsEmpty) return; // Table is not ready. + + // Allocate and initialize the XY table. + _xyTable = new ComputeBuffer(data.Length, sizeof(float)); + _xyTable.SetData(data); + } + + // Try retrieving the last frame. + var (color, depth) = _driver.LockLastFrame(); + if (color.IsEmpty || depth.IsEmpty) return; + + // Load the frame data into the temporary textures. + _temporaries.color.LoadRawTextureData(color.Span); + _temporaries.depth.LoadRawTextureData(depth.Span); + _temporaries.color.Apply(); + _temporaries.depth.Apply(); + + // We don't need the last frame any more. + _driver.ReleaseLastFrame(); + + // Invoke the unprojection shader. + _material.SetTexture("_ColorTexture", _temporaries.color); + _material.SetTexture("_DepthTexture", _temporaries.depth); + _material.SetBuffer("_XYTable", _xyTable); + _material.SetFloat("_MaxDepth", _deviceSettings.maxDepth); + var prevRT = RenderTexture.active; + GraphicsExtensions.SetRenderTarget(_colorTexture, _positionTexture); + Graphics.Blit(null, _material, 0); + RenderTexture.active = prevRT; + } + + #endregion + } +} diff --git a/UnityProject/Assets/ThridParty/Keijiro/PointCloudBaker.cs.meta b/UnityProject/Assets/ThridParty/Keijiro/PointCloudBaker.cs.meta new file mode 100644 index 0000000..d5733ce --- /dev/null +++ b/UnityProject/Assets/ThridParty/Keijiro/PointCloudBaker.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 48e1b0f548fe9b74c90eb10da5cffd13 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/UnityProject/Packages/manifest.json b/UnityProject/Packages/manifest.json index b61af9f..71c6442 100644 --- a/UnityProject/Packages/manifest.json +++ b/UnityProject/Packages/manifest.json @@ -1,6 +1,14 @@ { - -"dependencies": { + "scopedRegistries": [ + { + "name": "Unity NuGet", + "url": "https://unitynuget-registry.azurewebsites.net", + "scopes": [ + "org.nuget" + ] + } + ], + "dependencies": { "com.unity.2d.sprite": "1.0.0", "com.unity.2d.tilemap": "1.0.0", "com.unity.ads": "2.0.8", @@ -19,6 +27,7 @@ "com.unity.timeline": "1.1.0", "com.unity.ugui": "1.0.0", "com.unity.visualeffectgraph": "6.9.2-preview", + "org.nuget.system.memory": "4.5.3", "com.unity.modules.ai": "1.0.0", "com.unity.modules.androidjni": "1.0.0", "com.unity.modules.animation": "1.0.0", diff --git a/UnityProject/ProjectSettings/QualitySettings.asset b/UnityProject/ProjectSettings/QualitySettings.asset index 40cfd76..b70de7b 100644 --- a/UnityProject/ProjectSettings/QualitySettings.asset +++ b/UnityProject/ProjectSettings/QualitySettings.asset @@ -216,4 +216,5 @@ QualitySettings: asyncUploadPersistentBuffer: 1 resolutionScalingFixedDPIFactor: 1 excludedTargetPlatforms: [] - m_PerPlatformDefaultQuality: {} + m_PerPlatformDefaultQuality: + Standalone: 0