Integrate Keijiro's code

UnityProject/Assembly-CSharp-Editor.csproj

@@ -76,6 +76,7 @@
     <Compile Include="Assets\Editor\UnitTest\UnitTest_SpectrumGeneratorEditMode.cs" />
     <Compile Include="Assets\Editor\UnitTest\UnitTest_SpectrumGeneratorPlayMode.cs" />
     <Compile Include="Assets\ThridParty\Editor\KinectCopyPluginDataHelper.cs" />
+    <Compile Include="Assets\ThridParty\Keijiro\Editor\DeviceSettingsEditor.cs" />
     <Reference Include="UnityEditor.TestRunner">
       <HintPath>C:/Users/chikashi/Development/Soundvision/UnityProject/Library/ScriptAssemblies/UnityEditor.TestRunner.dll</HintPath>
     </Reference>

UnityProject/Assembly-CSharp.csproj

@@ -139,6 +139,13 @@
     <Compile Include="Assets\Scripts\Visualizer\TexturePanelBehaviour.cs" />
     <Compile Include="Assets\Scripts\Visualizer\WaterfallVisualizer.cs" />
     <Compile Include="Assets\Scripts\Visualizer\WaveformVisualizerBehaviour.cs" />
+    <Compile Include="Assets\ThridParty\Keijiro\DeviceSettings.cs" />
+    <Compile Include="Assets\ThridParty\Keijiro\Internal\DeviceSettingController.cs" />
+    <Compile Include="Assets\ThridParty\Keijiro\Internal\GraphicsExtensions.cs" />
+    <Compile Include="Assets\ThridParty\Keijiro\Internal\K4aExtensions.cs" />
+    <Compile Include="Assets\ThridParty\Keijiro\Internal\ThreadedDriver.cs" />
+    <Compile Include="Assets\ThridParty\Keijiro\Internal\XYTable.cs" />
+    <Compile Include="Assets\ThridParty\Keijiro\PointCloudBaker.cs" />
     <Compile Include="Assets\ThridParty\KinectScript\CameraIntrinsics.cs" />
     <Compile Include="Assets\ThridParty\KinectScript\CollectionMap.cs" />
     <Compile Include="Assets\ThridParty\KinectScript\EventPump.cs" />
@@ -237,6 +244,7 @@
     <Compile Include="Assets\Tiling.cs" />
     <None Include="Assets\Resources\Shader\Difference.compute" />
     <None Include="Assets\ThridParty\Assets\Construction_Site_Column\README_Import_to_2018-HDRP.txt" />
+    <None Include="Assets\ThridParty\Keijiro\Internal\Unproject.shader" />
     <None Include="Assets\ThridParty\Assets\JapaneseHouse\ReadMe.txt" />
     <None Include="Assets\Resources\buildNumber.txt" />
     <None Include="Assets\Third-Party-Notices.txt" />

UnityProject/Assets/RenderTextures.meta

@@ -1,5 +1,5 @@
 fileFormatVersion: 2
-guid: 7818a8ea7d4c62f4d8e6d0aa6828b937
+guid: a727511980477174bb68636438f49427
 folderAsset: yes
 DefaultImporter:
   externalObjects: {}

UnityProject/Assets/RenderTextures/Color.renderTexture

@@ -0,0 +1,37 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!84 &8400000
+RenderTexture:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_Name: Color
+  m_ImageContentsHash:
+    serializedVersion: 2
+    Hash: 00000000000000000000000000000000
+  m_ForcedFallbackFormat: 4
+  m_DownscaleFallback: 0
+  serializedVersion: 3
+  m_Width: 2048
+  m_Height: 1536
+  m_AntiAliasing: 1
+  m_MipCount: -1
+  m_DepthFormat: 0
+  m_ColorFormat: 8
+  m_MipMap: 0
+  m_GenerateMips: 1
+  m_SRGB: 0
+  m_UseDynamicScale: 0
+  m_BindMS: 0
+  m_EnableCompatibleFormat: 1
+  m_TextureSettings:
+    serializedVersion: 2
+    m_FilterMode: 0
+    m_Aniso: 0
+    m_MipBias: 0
+    m_WrapU: 1
+    m_WrapV: 1
+    m_WrapW: 1
+  m_Dimension: 2
+  m_VolumeDepth: 1

UnityProject/Assets/RenderTextures/Color.renderTexture.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 046aa26a9af8e164eb0b20672983a11f
+NativeFormatImporter:
+  externalObjects: {}
+  mainObjectFileID: 0
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/RenderTextures/Position.renderTexture

@@ -0,0 +1,37 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!84 &8400000
+RenderTexture:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_Name: Position
+  m_ImageContentsHash:
+    serializedVersion: 2
+    Hash: 00000000000000000000000000000000
+  m_ForcedFallbackFormat: 4
+  m_DownscaleFallback: 0
+  serializedVersion: 3
+  m_Width: 2048
+  m_Height: 1536
+  m_AntiAliasing: 1
+  m_MipCount: -1
+  m_DepthFormat: 0
+  m_ColorFormat: 48
+  m_MipMap: 0
+  m_GenerateMips: 1
+  m_SRGB: 0
+  m_UseDynamicScale: 0
+  m_BindMS: 0
+  m_EnableCompatibleFormat: 1
+  m_TextureSettings:
+    serializedVersion: 2
+    m_FilterMode: 0
+    m_Aniso: 0
+    m_MipBias: 0
+    m_WrapU: 1
+    m_WrapV: 1
+    m_WrapW: 1
+  m_Dimension: 2
+  m_VolumeDepth: 1

UnityProject/Assets/RenderTextures/Position.renderTexture.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: f235609b825377142ab45895e91b6839
+NativeFormatImporter:
+  externalObjects: {}
+  mainObjectFileID: 0
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/Settings.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: c88789b858a250f4fb98c3d6eab7a086
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/Settings/DeviceSetting.asset

@@ -0,0 +1,26 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!114 &11400000
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: daa4216d09a83dc479a7ee221b2a5bf9, type: 3}
+  m_Name: Akvfx Settings
+  m_EditorClassIdentifier: 
+  _autoExposure: 1
+  _exposure: 0.5
+  _autoWhiteBalance: 1
+  _whiteBalance: 0.5
+  _brightness: 0.5
+  _contrast: 0.5
+  _saturation: 0.5
+  _sharpness: 0.5
+  _gain: 1
+  _enableBlc: 0
+  _powerIs60Hz: 1
+  _maxDepth: 1

UnityProject/Assets/Settings/DeviceSetting.asset.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: f8122729f4736dd4ab99617ee19feec3
+NativeFormatImporter:
+  externalObjects: {}
+  mainObjectFileID: 11400000
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/ThridParty/Keijiro.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 42ba717c0afb14b4294df4939f72246d
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/ThridParty/Keijiro/DeviceSettings.cs

@@ -0,0 +1,136 @@
+using UnityEngine;
+
+namespace Akvfx
+{
+    public sealed class DeviceSettings : ScriptableObject
+    {
+        #region Editable fields
+
+        [SerializeField] bool _autoExposure = true;
+        [SerializeField, Range(0, 1)] float _exposure = 0.5f;
+
+        [SerializeField] bool _autoWhiteBalance = true;
+        [SerializeField, Range(0, 1)] float _whiteBalance = 0.5f;
+
+        [SerializeField, Range(0, 1)] float _brightness = 0.5f;
+        [SerializeField, Range(0, 1)] float _contrast = 0.5f;
+        [SerializeField, Range(0, 1)] float _saturation = 0.5f;
+        [SerializeField, Range(0, 1)] float _sharpness = 0.5f;
+        [SerializeField, Range(0, 1)] float _gain = 1;
+
+        [SerializeField] bool _enableBlc = false;
+        [SerializeField] bool _powerIs60Hz = true;
+
+        [SerializeField, Range(0, 6.6f)] float _maxDepth = 1;
+
+        #endregion
+
+        #region Public accessors
+
+        public bool autoExposure {
+            get { return _autoExposure; }
+            set { _autoExposure = value; }
+        }
+
+        public float exposure {
+            get { return _exposure; }
+            set { _exposure = value; }
+        }
+
+        public bool autoWhiteBalance {
+            get { return _autoWhiteBalance; }
+            set { _autoWhiteBalance = value; }
+        }
+
+        public float whiteBalance {
+            get { return _whiteBalance; }
+            set { _whiteBalance = value; }
+        }
+
+        public float brightness {
+            get { return _brightness; }
+            set { _brightness = value; }
+        }
+
+        public float contrast {
+            get { return _contrast; }
+            set { _contrast = value; }
+        }
+
+        public float saturation {
+            get { return _saturation; }
+            set { _saturation = value; }
+        }
+
+        public float sharpness {
+            get { return _sharpness; }
+            set { _sharpness = value; }
+        }
+
+        public float gain {
+            get { return _gain; }
+            set { _gain = value; }
+        }
+
+        public bool enableBlc {
+            get { return _enableBlc; }
+            set { _enableBlc = value; }
+        }
+
+        public bool powerIs60Hz {
+            get { return _powerIs60Hz; }
+            set { _powerIs60Hz = value; }
+        }
+
+        public float maxDepth {
+            get { return _maxDepth; }
+            set { _maxDepth = value; }
+        }
+
+        #endregion
+
+        #region Internal properties
+
+        internal int ExposureDeviceValue { get {
+            if (_autoExposure) return -1;
+            var exp = Mathf.Pow(_exposure, 8);
+            return (int)Mathf.Lerp(488.0f, 1000000.0f, exp);
+        } }
+
+        internal int WhiteBalanceDeviceValue { get {
+            if (_autoWhiteBalance) return -1;
+            var x = (int)Mathf.Lerp(2500, 10000, _whiteBalance);
+            return x - x % 10; // Should be divisible by 10.
+        } }
+
+        internal int BrightnessDeviceValue { get {
+            return (int)Mathf.Lerp(0, 255, _brightness);
+        } }
+
+        internal int ContrastDeviceValue { get {
+            return (int)Mathf.Lerp(0, 10, _contrast);
+        } }
+
+        internal int SaturationDeviceValue { get {
+            return (int)Mathf.Lerp(0, 63, _saturation);
+        } }
+
+        internal int SharpnessDeviceValue { get {
+            return (int)Mathf.Lerp(0, 4, _sharpness);
+        } }
+
+        internal int GainDeviceValue { get {
+            return (int)Mathf.Lerp(0, 255, _gain);
+        } }
+
+        internal int BlcDeviceValue { get {
+            return _enableBlc ? 1 : 0;
+        } }
+
+        internal int PowerFreqDeviceValue { get {
+            return _powerIs60Hz ? 2 : 1;
+        } }
+
+        #endregion
+    }
+}

UnityProject/Assets/ThridParty/Keijiro/DeviceSettings.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: daa4216d09a83dc479a7ee221b2a5bf9
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/ThridParty/Keijiro/Editor.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 9aff79d67109d4f4cb7dd61f35f6e452
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/ThridParty/Keijiro/Editor/DeviceSettingsEditor.cs

@@ -0,0 +1,104 @@
+using UnityEngine;
+using UnityEditor;
+
+namespace Akvfx
+{
+    [CanEditMultipleObjects]
+    [CustomEditor(typeof(DeviceSettings))]
+    sealed class DeviceSettingsEditor : Editor
+    {
+        SerializedProperty _autoExposure;
+        SerializedProperty _exposure;
+
+        SerializedProperty _autoWhiteBalance;
+        SerializedProperty _whiteBalance;
+
+        SerializedProperty _brightness;
+        SerializedProperty _contrast;
+        SerializedProperty _saturation;
+        SerializedProperty _sharpness;
+        SerializedProperty _gain;
+
+        SerializedProperty _enableBlc;
+        SerializedProperty _powerIs60Hz;
+
+        SerializedProperty _maxDepth;
+
+        static readonly (
+            GUIContent enableBlc,
+            GUIContent powerIs60Hz
+        ) _labels = (
+            new GUIContent("Enable BLC"),
+            new GUIContent("Power is 60Hz")
+        );
+
+        void OnEnable()
+        {
+            _autoExposure = serializedObject.FindProperty("_autoExposure");
+            _exposure = serializedObject.FindProperty("_exposure");
+
+            _autoWhiteBalance = serializedObject.FindProperty("_autoWhiteBalance");
+            _whiteBalance = serializedObject.FindProperty("_whiteBalance");
+
+            _brightness = serializedObject.FindProperty("_brightness");
+            _contrast = serializedObject.FindProperty("_contrast");
+            _saturation = serializedObject.FindProperty("_saturation");
+            _sharpness = serializedObject.FindProperty("_sharpness");
+            _gain = serializedObject.FindProperty("_gain");
+
+            _enableBlc = serializedObject.FindProperty("_enableBlc");
+            _powerIs60Hz = serializedObject.FindProperty("_powerIs60Hz");
+
+            _maxDepth = serializedObject.FindProperty("_maxDepth");
+        }
+
+        public override void OnInspectorGUI()
+        {
+            serializedObject.Update();
+
+            EditorGUILayout.PropertyField(_autoExposure);
+            if (_autoExposure.hasMultipleDifferentValues ||
+                !_autoExposure.boolValue)
+            {
+                EditorGUI.indentLevel++;
+                EditorGUILayout.PropertyField(_exposure);
+                EditorGUI.indentLevel--;
+            }
+
+            EditorGUILayout.PropertyField(_autoWhiteBalance);
+            if (_autoWhiteBalance.hasMultipleDifferentValues ||
+                !_autoWhiteBalance.boolValue)
+            {
+                EditorGUI.indentLevel++;
+                EditorGUILayout.PropertyField(_whiteBalance);
+                EditorGUI.indentLevel--;
+            }
+
+            EditorGUILayout.PropertyField(_brightness);
+            EditorGUILayout.PropertyField(_contrast);
+            EditorGUILayout.PropertyField(_saturation);
+            EditorGUILayout.PropertyField(_sharpness);
+            EditorGUILayout.PropertyField(_gain);
+
+            EditorGUILayout.PropertyField(_enableBlc, _labels.enableBlc);
+            EditorGUILayout.PropertyField(_powerIs60Hz, _labels.powerIs60Hz);
+
+            EditorGUILayout.PropertyField(_maxDepth);
+
+            serializedObject.ApplyModifiedProperties();
+        }
+
+        [MenuItem("Assets/Create/Akvfx/Device Settings")]
+        public static void CreateDeviceSettings()
+        {
+            var asset = ScriptableObject.CreateInstance<DeviceSettings>();
+
+            AssetDatabase.CreateAsset(asset, "Assets/Akvfx Settings.asset");
+            AssetDatabase.SaveAssets();
+
+            EditorUtility.FocusProjectWindow();
+
+            Selection.activeObject = asset;
+        }
+    }
+}

UnityProject/Assets/ThridParty/Keijiro/Editor/DeviceSettingsEditor.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 39b37077152d5024ab6ca873d5de277f
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/ThridParty/Keijiro/Internal.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: 84f2346cb4622004499b20cda8c2bee0
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/ThridParty/Keijiro/Internal/DeviceSettingController.cs

@@ -0,0 +1,110 @@
+using Microsoft.Azure.Kinect.Sensor;
+
+namespace Akvfx
+{
+    sealed class DeviceSettingController
+    {
+        #region Public methods
+
+        public DeviceSettingController(Device device, DeviceSettings initial)
+        {
+            ApplyInternal(device, initial, true);
+        }
+
+        public void ApplySettings(Device device, DeviceSettings settings)
+        {
+            ApplyInternal(device, settings, false);
+        }
+
+        #endregion
+
+        #region Cached control values
+
+        int _exposure;
+        int _whiteBalance;
+        int _brightness;
+        int _contrast;
+        int _saturation;
+        int _sharpness;
+        int _gain;
+        int _blc;
+        int _powerFreq;
+
+        #endregion
+
+        #region Private methods
+
+        int ApplyControl(
+            Device device, ColorControlCommand command,
+            int newValue, int prevValue, bool forceApply
+        )
+        {
+            // If nothing was changed, simply return the previous value.
+            if (!forceApply && newValue == prevValue) return prevValue;
+
+            // Apply the new value to the control.
+            device.SetColorControl(
+                command,
+                newValue < 0 ? ColorControlMode.Auto : ColorControlMode.Manual,
+                newValue < 0 ? 0 : newValue
+            );
+            return newValue;
+        }
+
+        void ApplyInternal(Device device, DeviceSettings settings, bool forceApply)
+        {
+            _exposure = ApplyControl(
+                device, ColorControlCommand.ExposureTimeAbsolute,
+                settings.ExposureDeviceValue, _exposure, forceApply
+            );
+
+            _whiteBalance = ApplyControl(
+                device, ColorControlCommand.Whitebalance,
+                settings.WhiteBalanceDeviceValue, _whiteBalance, forceApply
+            );
+
+            _brightness = ApplyControl(
+                device, ColorControlCommand.Brightness,
+                settings.BrightnessDeviceValue, _brightness, forceApply
+            );
+
+            _contrast = ApplyControl(
+                device, ColorControlCommand.Contrast,
+                settings.ContrastDeviceValue, _contrast, forceApply
+            );
+
+            _saturation = ApplyControl(
+                device, ColorControlCommand.Saturation,
+                settings.SaturationDeviceValue, _saturation, forceApply
+            );
+
+            _sharpness = ApplyControl(
+                device, ColorControlCommand.Sharpness,
+                settings.SharpnessDeviceValue, _sharpness, forceApply
+            );
+
+            // This is not documented, but the gain parameter can't update
+            // while the auto exposure is enabled. To delay updates, we do a
+            // bit tricky thing here.
+            if (_exposure < 0 || forceApply)
+                _gain = -1;
+            else
+                _gain = ApplyControl(
+                    device, ColorControlCommand.Gain,
+                    settings.GainDeviceValue, _gain, false
+                );
+
+            _blc = ApplyControl(
+                device, ColorControlCommand.BacklightCompensation,
+                settings.BlcDeviceValue, _blc, forceApply
+            );
+
+            _powerFreq = ApplyControl(
+                device, ColorControlCommand.PowerlineFrequency,
+                settings.PowerFreqDeviceValue, _powerFreq, forceApply
+            );
+        }
+
+        #endregion
+    }
+}

UnityProject/Assets/ThridParty/Keijiro/Internal/DeviceSettingController.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 492dfb729c0136d48b2917cc16811e38
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/ThridParty/Keijiro/Internal/GraphicsExtensions.cs

@@ -0,0 +1,75 @@
+//
+// This file contains some utility extensions for Unity graphics classes.
+// These extensions are mainly for adding Span support to storage classes.
+//
+
+using UnityEngine;
+using System;
+using System.Reflection;
+
+namespace Akvfx
+{
+    static class GraphicsExtensions
+    {
+        // MRT with tow render textures
+        public static void SetRenderTarget(RenderTexture rt1, RenderTexture rt2)
+        {
+            _target2[0] = rt1.colorBuffer;
+            _target2[1] = rt2.colorBuffer;
+            Graphics.SetRenderTarget(_target2, rt1.depthBuffer);
+        }
+
+        static RenderBuffer [] _target2 = new RenderBuffer[2];
+    }
+
+    static class Texture2DExtensions
+    {
+        // LoadRawTextureData with ReadOnlySpan
+        public unsafe static void LoadRawTextureData
+            (this Texture2D texture, ReadOnlySpan<byte> data)
+        {
+            fixed (byte* pData = &data.GetPinnableReference())
+                texture.LoadRawTextureData((IntPtr)pData, data.Length);
+        }
+    }
+
+    static class ComputeBufferExtensions
+    {
+        // SetData with ReadOnlySpan
+        public unsafe static void SetData<T>
+            (this ComputeBuffer buffer, ReadOnlySpan<T> data)
+            where T : unmanaged
+        {
+            fixed (T* pData = &data.GetPinnableReference())
+                buffer.SetData((IntPtr)pData, data.Length, sizeof(T));
+        }
+
+        // Directly load an unmanaged memory block to a compute buffer via an
+        // Intptr. This is not a public interface so will be broken one day.
+        // DO NOT TRY AT HOME.
+        public static void SetData
+            (this ComputeBuffer buffer, IntPtr pointer, int count, int stride)
+        {
+            if (_method == null)
+            {
+                _method = typeof(ComputeBuffer).GetMethod(
+                    "InternalSetNativeData",
+                    BindingFlags.InvokeMethod |
+                    BindingFlags.NonPublic |
+                    BindingFlags.Instance
+                );
+            }
+
+            _args5[0] = pointer;
+            _args5[1] = 0;      // source offset
+            _args5[2] = 0;      // buffer offset
+            _args5[3] = count;
+            _args5[4] = stride;
+
+            _method.Invoke(buffer, _args5);
+        }
+
+        static MethodInfo _method;
+        static object [] _args5 = new object[5];
+    }
+}

UnityProject/Assets/ThridParty/Keijiro/Internal/GraphicsExtensions.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 01b4ad67ec4d57f4d958129b1d240d75
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/ThridParty/Keijiro/Internal/K4aExtensions.cs

@@ -0,0 +1,29 @@
+using System.Reflection;
+
+namespace Akvfx
+{
+    static class K4aExtensions
+    {
+        // Set false to Allocator.SafeCopyNativeBuffers. You can earn a few
+        // milliseconds by skipping safe-copy. Note that it turns minor bugs
+        // undebuggable crashes.
+        public static void DisableSafeCopyNativeBuffers()
+        {
+            var allocator = System.Type.GetType(
+                "Microsoft.Azure.Kinect.Sensor.Allocator,Microsoft.Azure.Kinect.Sensor"
+            );
+
+            var singleton = allocator.GetProperty(
+                "Singleton",
+                BindingFlags.Public | BindingFlags.Static
+            );
+
+            var safeCopyNativeBuffers = allocator.GetProperty(
+                "SafeCopyNativeBuffers",
+                BindingFlags.Public | BindingFlags.Instance
+            );
+
+            safeCopyNativeBuffers.SetValue(singleton.GetValue(null), false);
+        }
+    }
+}

UnityProject/Assets/ThridParty/Keijiro/Internal/K4aExtensions.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 275db18c032614049a8e1b71387b610a
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/ThridParty/Keijiro/Internal/ThreadedDriver.cs

@@ -0,0 +1,150 @@
+using Microsoft.Azure.Kinect.Sensor;
+using System;
+using System.Collections.Concurrent;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Akvfx
+{
+    sealed class ThreadedDriver : IDisposable
+    {
+        #region Public properties and methods
+
+        public ThreadedDriver(DeviceSettings settings)
+        {
+            // FIXME: Dangerous. We should do this only on Player.
+            K4aExtensions.DisableSafeCopyNativeBuffers();
+
+            _settings = settings;
+
+            _captureThread = new Thread(CaptureThread);
+            _captureThread.Start();
+        }
+
+        public void Dispose()
+        {
+            _terminate = true;
+            _captureThread.Join();
+
+            TrimQueue(0);
+            ReleaseLastFrame();
+
+            GC.SuppressFinalize(this);
+        }
+
+        public ReadOnlySpan<float> XYTable
+        {
+            get { return _xyTable != null ? _xyTable.Data : null; }
+        }
+
+        public (ReadOnlyMemory<byte> color,
+                ReadOnlyMemory<byte> depth) LockLastFrame()
+        {
+            // Try retrieving the last frame.
+            if (_lockedFrame.capture == null)
+                _queue.TryDequeue(out _lockedFrame);
+
+            // Return null if it failed to retrieve.
+            if (_lockedFrame.capture == null) return (null, null);
+
+            return (_lockedFrame.capture.Color.Memory,
+                    _lockedFrame.depth.Memory);
+        }
+
+        public void ReleaseLastFrame()
+        {
+            _lockedFrame.capture?.Dispose();
+            _lockedFrame.depth?.Dispose();
+            _lockedFrame = (null, null);
+        }
+
+        #endregion
+
+        #region Private objects
+
+        DeviceSettings _settings;
+        XYTable _xyTable;
+
+        #endregion
+
+        #region Capture queue
+
+        ConcurrentQueue<(Capture capture, Image depth)>
+            _queue = new ConcurrentQueue<(Capture, Image)>();
+
+        (Capture capture, Image depth) _lockedFrame;
+
+        // Trim the queue to a specified count.
+        void TrimQueue(int count)
+        {
+            while (_queue.Count > count)
+            {
+                (Capture capture, Image depth) temp;
+                _queue.TryDequeue(out temp);
+                temp.capture?.Dispose();
+                temp.depth?.Dispose();
+            }
+        }
+
+        #endregion
+
+        #region Capture thread
+
+        Thread _captureThread;
+        bool _terminate;
+
+        void CaptureThread()
+        {
+            // If there is no available device, do nothing.
+            if (Device.GetInstalledCount() == 0) return;
+
+            // Open the default device.
+            var device = Device.Open();
+
+            // Start capturing with custom settings.
+            device.StartCameras(
+                new DeviceConfiguration {
+                    ColorFormat = ImageFormat.ColorBGRA32,
+                    ColorResolution = ColorResolution.R1536p, // 2048 x 1536 (4:3)
+                    DepthMode = DepthMode.NFOV_Unbinned,      // 640x576
+                    SynchronizedImagesOnly = true
+                }
+            );
+
+            // Construct XY table as a background task.
+            Task.Run(() => {
+                _xyTable = new XYTable(device.GetCalibration(), 2048, 1536);
+            });
+
+            // Set up the transformation object.
+            var transformation = new Transformation(device.GetCalibration());
+
+            // Initially apply the device settings.
+            var setter = new DeviceSettingController(device, _settings);
+
+            while (!_terminate)
+            {
+                // Get a frame capture.
+                var capture = device.GetCapture();
+
+                // Transform the depth image to the color perspective.
+                var depth = transformation.DepthImageToColorCamera(capture);
+
+                // Push the frame to the capture queue.
+                _queue.Enqueue((capture, depth));
+
+                // Remove old frames.
+                TrimQueue(1);
+
+                // Apply changes on the device settings.
+                setter.ApplySettings(device, _settings);
+            }
+
+            // Cleaning up.
+            transformation.Dispose();
+            device.Dispose();
+        }
+
+        #endregion
+    }
+}

UnityProject/Assets/ThridParty/Keijiro/Internal/ThreadedDriver.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: b43ac31f3a29da64ba703a0bcf364a95
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/ThridParty/Keijiro/Internal/Unproject.shader

@@ -0,0 +1,67 @@
+Shader "Hidden/Akvfx/Unproject"
+{
+    CGINCLUDE
+
+    #include "UnityCG.cginc"
+
+    texture2D _ColorTexture;
+    texture2D _DepthTexture;
+    StructuredBuffer<float> _XYTable;
+    float _MaxDepth;
+
+    void Vertex(
+        float4 position : POSITION,
+        out float4 positionOut : SV_Position,
+        inout float2 texCoord : TEXCOORD0
+    )
+    {
+        positionOut = UnityObjectToClipPos(position);
+    }
+
+    void Fragment(
+        float4 position : SV_Position,
+        float2 texCoord : TEXCOORD0,
+        out float4 colorOut : SV_Target0,
+        out float4 positionOut : SV_Target1
+    )
+    {
+        uint w, h;
+        _ColorTexture.GetDimensions(w, h);
+
+        // Texture index
+        uint tx = texCoord.x * w;
+        uint ty = texCoord.y * h;
+
+        // Color sample
+        float4 color = _ColorTexture[uint2(tx, ty)];
+
+        // Depth sample (int16 -> float)
+        int d0 = _DepthTexture[uint2(tx * 2 + 0, ty)] * 255;
+        int d1 = _DepthTexture[uint2(tx * 2 + 1, ty)] * 255;
+        float depth = (float)(d0 + (d1 << 8)) / 1000;
+        float mask = depth > 0 && depth < _MaxDepth;
+        float z = lerp(_MaxDepth, depth, mask);
+
+        // XY table lookup
+        uint xy_i = (tx + ty * w) * 2;
+        float2 xy = float2(_XYTable[xy_i], -_XYTable[xy_i + 1]);
+
+        // MRT output write
+        colorOut = float4(color.rgb, mask);
+        positionOut = float4(xy * z, z, mask);
+    }
+
+    ENDCG
+
+    SubShader
+    {
+        Cull Off ZWrite Off ZTest Always
+        Pass
+        {
+            CGPROGRAM
+            #pragma vertex Vertex
+            #pragma fragment Fragment
+            ENDCG
+        }
+    }
+}

UnityProject/Assets/ThridParty/Keijiro/Internal/Unproject.shader.meta

@@ -0,0 +1,9 @@
+fileFormatVersion: 2
+guid: c2bfd9668998b6848ac60594704c344c
+ShaderImporter:
+  externalObjects: {}
+  defaultTextures: []
+  nonModifiableTextures: []
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/ThridParty/Keijiro/Internal/XYTable.cs

@@ -0,0 +1,80 @@
+using Microsoft.Azure.Kinect.Sensor;
+using System;
+using System.Runtime.InteropServices;
+using System.Threading.Tasks;
+
+namespace Akvfx
+{
+    //
+    // Per pixel ray direction table ("XY table") used to unproject depth
+    // samples to the 3D camera space
+    //
+    // This class directly invokes a native method (k4a_calibration_2d_to_3d)
+    // contained in libk4a to avoid double symbol definition problem between
+    // System.Numerics and System.Numerics.Vectors.
+    //
+    sealed class XYTable
+    {
+        // Public property: Table data
+        public ReadOnlySpan<float> Data { get { return _data; } }
+
+        // Float vvector types
+        struct Float2 { public float x, y; }
+        struct Float3 { public float x, y, z; }
+
+        // Data storage
+        float [] _data;
+
+        // Constructor
+        public XYTable(Calibration calibration, int width, int height)
+        {
+            // Data storage allocation
+            var table = new float [width * height * 2];
+
+            // XY table initialization
+            // These loops could take a significant amount of time (e.g. 1 sec)
+            // even on a decent desktop PC. Luckily, this can be easily
+            // parallelized by using Parallel.For, so we did.
+            Parallel.For(0, height, y => {
+                Float2 v2;
+                Float3 v3;
+                bool isValid;
+
+                v2.y = y;
+                var offs = width * 2 * y;
+
+                for (var x = 0; x < width; x++)
+                {
+                    v2.x = x;
+
+                    k4a_calibration_2d_to_3d(
+                        ref calibration,
+                        ref v2, 1,
+                        CalibrationDeviceType.Color,
+                        CalibrationDeviceType.Color,
+                        out v3,
+                        out isValid
+                    );
+
+                    table[offs++] = v3.x;
+                    table[offs++] = v3.y;
+                }
+            });
+
+            // Publish the table data.
+            _data = table;
+        }
+
+        // k4a_calibration_2d_to_3d native method from libk4a
+        [DllImport("k4a", CallingConvention = CallingConvention.Cdecl)]
+        static extern int k4a_calibration_2d_to_3d(
+            [In] ref Calibration calibration,
+            ref Float2 source_point2d,
+            float source_depth,
+            CalibrationDeviceType source_camera,
+            CalibrationDeviceType target_camera,
+            out Float3 target_point3d,
+            out bool valid
+        );
+    }
+}

UnityProject/Assets/ThridParty/Keijiro/Internal/XYTable.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: efd020f99d8831e4a9f43be37d1de501
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/Assets/ThridParty/Keijiro/PointCloudBaker.cs

@@ -0,0 +1,91 @@
+using UnityEngine;
+using GraphicsFormat = UnityEngine.Experimental.Rendering.GraphicsFormat;
+
+namespace Akvfx
+{
+    public sealed class PointCloudBaker : MonoBehaviour
+    {
+        #region Editable attributes
+
+        [SerializeField] DeviceSettings _deviceSettings = null;
+        [SerializeField] RenderTexture _colorTexture = null;
+        [SerializeField] RenderTexture _positionTexture = null;
+        [SerializeField] Shader _shader = null;
+
+        #endregion
+
+        #region Internal objects
+
+        ThreadedDriver _driver;
+        Material _material;
+        ComputeBuffer _xyTable;
+        (Texture2D color, Texture2D depth) _temporaries;
+
+        #endregion
+
+        #region MonoBehaviour implementation
+
+        void Start()
+        {
+            // Start capturing via the threaded driver.
+            _driver = new ThreadedDriver(_deviceSettings);
+
+            // Temporary objects for convertion shader
+            _material = new Material(_shader);
+            _temporaries = (
+                new Texture2D(2048, 1536, GraphicsFormat.B8G8R8A8_SRGB, 0),
+                new Texture2D(2048 * 2, 1536, GraphicsFormat.R8_UNorm, 0)
+            );
+        }
+
+        void OnDestroy()
+        {
+            if (_material != null) Destroy(_material);
+            _xyTable?.Dispose();
+            if (_temporaries.color != null) Destroy(_temporaries.color);
+            if (_temporaries.depth != null) Destroy(_temporaries.depth);
+            _driver?.Dispose();
+        }
+
+        RenderBuffer[] _mrt = new RenderBuffer[2];
+
+        unsafe void Update()
+        {
+            // Try initializing XY table if it's not ready.
+            if (_xyTable == null)
+            {
+                var data = _driver.XYTable;
+                if (data.IsEmpty) return; // Table is not ready.
+
+                // Allocate and initialize the XY table.
+                _xyTable = new ComputeBuffer(data.Length, sizeof(float));
+                _xyTable.SetData(data);
+            }
+
+            // Try retrieving the last frame.
+            var (color, depth) = _driver.LockLastFrame();
+            if (color.IsEmpty || depth.IsEmpty) return;
+
+            // Load the frame data into the temporary textures.
+            _temporaries.color.LoadRawTextureData(color.Span);
+            _temporaries.depth.LoadRawTextureData(depth.Span);
+            _temporaries.color.Apply();
+            _temporaries.depth.Apply();
+
+            // We don't need the last frame any more.
+            _driver.ReleaseLastFrame();
+
+            // Invoke the unprojection shader.
+            _material.SetTexture("_ColorTexture", _temporaries.color);
+            _material.SetTexture("_DepthTexture", _temporaries.depth);
+            _material.SetBuffer("_XYTable", _xyTable);
+            _material.SetFloat("_MaxDepth", _deviceSettings.maxDepth);
+            var prevRT = RenderTexture.active;
+            GraphicsExtensions.SetRenderTarget(_colorTexture, _positionTexture);
+            Graphics.Blit(null, _material, 0);
+            RenderTexture.active = prevRT;
+        }
+
+        #endregion
+    }
+}

UnityProject/Assets/ThridParty/Keijiro/PointCloudBaker.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: 48e1b0f548fe9b74c90eb10da5cffd13
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

UnityProject/ProjectSettings/QualitySettings.asset

@@ -216,4 +216,5 @@ QualitySettings:
     asyncUploadPersistentBuffer: 1
     resolutionScalingFixedDPIFactor: 1
     excludedTargetPlatforms: []
-  m_PerPlatformDefaultQuality: {}
+  m_PerPlatformDefaultQuality:
+    Standalone: 0