using UnityEngine; using System.Collections; using com.rfilkov.kinect; namespace com.rfilkov.components { /// /// SceneMeshRenderer renders virtual mesh of the environment in the scene, as detected by the given sensor. /// public class SceneMeshRenderer : MonoBehaviour { [Tooltip("Depth sensor index - 0 is the 1st one, 1 - the 2nd one, etc.")] public int sensorIndex = 0; [Tooltip("Resolution of the images used to generate the scene.")] private DepthSensorBase.PointCloudResolution sourceImageResolution = DepthSensorBase.PointCloudResolution.DepthCameraResolution; [Tooltip("Whether to show the mesh as point cloud, or as solid mesh.")] public bool showAsPointCloud = true; [Tooltip("Horizontal limit - minimum, in meters.")] [Range(-5f, 5f)] public float xMin = -2f; [Tooltip("Horizontal limit - maximum, in meters.")] [Range(-5f, 5f)] public float xMax = 2f; [Tooltip("Vertical limit - minimum, in meters.")] [Range(-5f, 5f)] public float yMin = -2f; [Tooltip("Vertical limit - maximum, in meters.")] [Range(-5f, 5f)] public float yMax = 2f; [Tooltip("Distance limit - minimum, in meters.")] [Range(0.5f, 10f)] public float zMin = 1f; [Tooltip("Distance limit - maximum, in meters.")] [Range(0.5f, 10f)] public float zMax = 3f; [Tooltip("Time interval between scene mesh updates, in seconds. 0 means no wait.")] public float updateMeshInterval = 0f; [Tooltip("Whether to include the detected players to the scene mesh or not.")] private bool includePlayers = true; [Tooltip("Whether to update the mesh, only when there are no players detected.")] private bool updateWhenNoPlayers = false; //[Tooltip("Whether to update the mesh collider as well, when the user mesh changes.")] //public bool updateMeshCollider = false; [Tooltip("Time interval between mesh-collider updates, in seconds. 0 means no mesh-collider updates.")] public float updateColliderInterval = 0f; // reference to object's mesh private Mesh mesh = null; // references to KM and data private KinectManager kinectManager = null; private KinectInterop.SensorData sensorData = null; private DepthSensorBase sensorInt = null; private Vector3 spaceScale = Vector3.zero; // render textures private RenderTexture colorTexture = null; private RenderTexture colorTextureCopy = null; private bool colorTextureCreated = false; // times private ulong lastSpaceCoordsTime = 0; private float lastMeshUpdateTime = 0f; private float lastColliderUpdateTime = 0f; private int minDepth = 0; private int maxDepth = 0; // image parameters private int imageWidth = 0; private int imageHeight = 0; // mesh parameters private Vector3[] spaceTable = null; private Vector3[] meshVertices = null; private int[] meshIndices = null; private byte[] meshVertUsed = null; private bool bMeshInited = false; void Start() { // get sensor data kinectManager = KinectManager.Instance; sensorData = (kinectManager != null && kinectManager.IsInitialized()) ? kinectManager.GetSensorData(sensorIndex) : null; } void OnDestroy() { if(bMeshInited) { // release the mesh-related resources FinishMesh(); } } void LateUpdate() { if(mesh == null && sensorData != null && sensorData.depthCamIntr != null) { // init mesh and its related data InitMesh(); } if (bMeshInited) { // min & max depth minDepth = Mathf.RoundToInt(zMin * 1000f); maxDepth = Mathf.RoundToInt(zMax * 1000f); // update the mesh UpdateMesh(); } } // inits the mesh and related data private void InitMesh() { // create mesh mesh = new Mesh { name = "SceneMesh-Sensor" + sensorIndex, indexFormat = UnityEngine.Rendering.IndexFormat.UInt32 }; MeshFilter meshFilter = GetComponent(); if(meshFilter != null) { meshFilter.mesh = mesh; } else { Debug.LogWarning("MeshFilter not found! You may not see the mesh on screen"); } if (sensorData != null && sensorData.sensorInterface != null) { sensorInt = (DepthSensorBase)sensorData.sensorInterface; Vector2Int imageRes = Vector2Int.zero; if (sensorInt.pointCloudColorTexture == null) { sensorInt.pointCloudResolution = sourceImageResolution; imageRes = sensorInt.GetPointCloudTexResolution(sensorData); colorTexture = KinectInterop.CreateRenderTexture(colorTexture, imageRes.x, imageRes.y, RenderTextureFormat.ARGB32); sensorInt.pointCloudColorTexture = colorTexture; colorTextureCreated = true; } else { imageRes = sensorInt.GetPointCloudTexResolution(sensorData); colorTexture = sensorInt.pointCloudColorTexture; colorTextureCreated = false; } // create copy texture colorTextureCopy = KinectInterop.CreateRenderTexture(colorTextureCopy, imageRes.x, imageRes.y, RenderTextureFormat.ARGB32); // set the color texture Renderer meshRenderer = GetComponent(); if (meshRenderer && meshRenderer.material && meshRenderer.material.mainTexture == null) { meshRenderer.material.mainTexture = colorTextureCopy; // sensorInt.pointCloudColorTexture; //meshRenderer.material.SetTextureScale("_MainTex", kinectManager.GetColorImageScale(sensorIndex)); } // image width & height imageWidth = imageRes.x; imageHeight = imageRes.y; int pointCount = imageWidth * imageHeight; // mesh arrays meshVertices = new Vector3[pointCount]; meshIndices = new int[6 * pointCount]; // 2 triangles per vertex, last row and column excluded meshVertUsed = new byte[pointCount]; spaceScale = kinectManager.GetSensorSpaceScale(sensorIndex); // create space table spaceTable = sensorInt.pointCloudResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution ? sensorInt.GetDepthCameraSpaceTable(sensorData) : sensorInt.GetColorCameraSpaceTable(sensorData); // init mesh uv array //spaceTable = new Vector3[pointCount]; Vector2[] meshUv = new Vector2[pointCount]; for (int y = 0, i = 0; y < imageHeight; y++) { for (int x = 0; x < imageWidth; x++, i++) { Vector2 imagePos = new Vector2(x, y); //spaceTable[i] = sensorInt.MapDepthPointToSpaceCoords(sensorData, imagePos, 1000); //sourceImageResolution == DepthSensorBase.PointCloudResolution.ColorCameraResolution ? //sensorInt.MapColorPointToSpaceCoords(sensorData, imagePos, 1000) : //sensorInt.MapDepthPointToSpaceCoords(sensorData, imagePos, 1000); meshUv[i] = new Vector2(imagePos.x / imageWidth, imagePos.y / imageHeight); } } mesh.vertices = meshVertices; mesh.SetIndices(meshIndices, MeshTopology.Triangles, 0); mesh.uv = meshUv; if(showAsPointCloud) { meshIndices = new int[pointCount]; for (int i = 0; i < pointCount; i++) meshIndices[i] = i; mesh.SetIndices(meshIndices, MeshTopology.Points, 0); } bMeshInited = true; } } // releases mesh-related resources private void FinishMesh() { if (sensorInt) { sensorInt.pointCloudColorTexture = null; } if (colorTexture && colorTextureCreated) { colorTexture.Release(); colorTexture = null; } if (colorTextureCopy) { colorTextureCopy.Release(); colorTextureCopy = null; } spaceTable = null; meshVertices = null; meshIndices = null; meshVertUsed = null; bMeshInited = false; } // updates the mesh according to current depth frame private void UpdateMesh() { if (bMeshInited && sensorData.depthImage != null && lastSpaceCoordsTime != sensorData.lastDepthFrameTime) { // check for players bool bHavePlayers = kinectManager.GetUsersCount() != 0; if ((Time.time - lastMeshUpdateTime) >= updateMeshInterval && (!updateWhenNoPlayers || !bHavePlayers)) { lastSpaceCoordsTime = sensorData.lastDepthFrameTime; lastMeshUpdateTime = Time.time; int imageWidth1 = imageWidth - 1, imageHeight1 = imageHeight - 1; const int maxDistMm = 200; // max distance between vertices in a triangle, in mm // copy the texture Graphics.CopyTexture(colorTexture, colorTextureCopy); for (int y = 0, di = 0, ti = 0; y < imageHeight; y++) { for (int x = 0; x < imageWidth; x++, di++, ti += 6) { ushort depth = sensorData.depthImage[di]; bool isValidBodyPixel = !includePlayers && bHavePlayers ? sensorData.bodyIndexImage[di] == 255 : true; bool bVertexSet = false; if (depth >= minDepth && depth <= maxDepth && isValidBodyPixel) { float fDepth = (float)depth * 0.001f; Vector3 vVertex = new Vector3(spaceTable[di].x * fDepth * spaceScale.x, spaceTable[di].y * fDepth * spaceScale.y, fDepth); bool bUsedVertex = !showAsPointCloud && meshVertUsed[di] != 0; if (bUsedVertex || (vVertex.x >= xMin && vVertex.x <= xMax && vVertex.y >= yMin && vVertex.y <= yMax)) { meshVertices[di] = vVertex; if (!showAsPointCloud) { meshVertUsed[di] = 1; if (x < imageWidth1 && y < imageHeight1) { int tl = di; ushort tld = depth; int tr = tl + 1; ushort trd = sensorData.depthImage[tr]; int bl = tl + imageWidth; ushort bld = sensorData.depthImage[bl]; int br = bl + 1; ushort brd = sensorData.depthImage[br]; float fBrDepth = (float)brd * 0.001f; Vector3 vVertex2 = new Vector3(spaceTable[br].x * fBrDepth * spaceScale.x, spaceTable[di].y * fBrDepth * spaceScale.y, fBrDepth); // 1st triangle if (tld >= minDepth && tld <= maxDepth && trd >= minDepth && trd <= maxDepth && bld >= minDepth && bld <= maxDepth && vVertex2.x >= xMin && vVertex2.x <= xMax && vVertex2.y >= yMin && vVertex2.y <= yMax && Mathf.Abs(trd - tld) < maxDistMm && Mathf.Abs(bld - tld) < maxDistMm) { meshIndices[ti] = bl; meshIndices[ti + 1] = tr; meshIndices[ti + 2] = tl; meshVertUsed[trd] = meshVertUsed[bld] = 1; } else { meshIndices[ti] = meshIndices[ti + 1] = meshIndices[ti + 2] = 0; meshVertUsed[trd] = meshVertUsed[bld] = 0; } // 2nd triangle if (bld >= minDepth && bld <= maxDepth && trd >= minDepth && trd <= maxDepth && brd >= minDepth && brd <= maxDepth && vVertex2.x >= xMin && vVertex2.x <= xMax && vVertex2.y >= yMin && vVertex2.y <= yMax && Mathf.Abs(trd - bld) < maxDistMm && Mathf.Abs(brd - bld) < maxDistMm) { meshIndices[ti + 3] = br; meshIndices[ti + 4] = tr; meshIndices[ti + 5] = bl; meshVertUsed[bld] = meshVertUsed[trd] = meshVertUsed[brd] = 1; } else { meshIndices[ti + 3] = meshIndices[ti + 4] = meshIndices[ti + 5] = 0; meshVertUsed[bld] = meshVertUsed[trd] = meshVertUsed[brd] = 0; } } else { meshIndices[ti] = meshIndices[ti + 1] = meshIndices[ti + 2] = 0; meshIndices[ti + 3] = meshIndices[ti + 4] = meshIndices[ti + 5] = 0; } } bVertexSet = true; } } if (!bVertexSet) { meshVertices[di] = Vector3.zero; if (!showAsPointCloud) { meshIndices[ti] = meshIndices[ti + 1] = meshIndices[ti + 2] = 0; meshIndices[ti + 3] = meshIndices[ti + 4] = meshIndices[ti + 5] = 0; } } } } mesh.vertices = meshVertices; if(!showAsPointCloud) { mesh.SetIndices(meshIndices, MeshTopology.Triangles, 0); } mesh.RecalculateBounds(); if (updateColliderInterval > 0 && (Time.time - lastColliderUpdateTime) >= updateColliderInterval) { lastColliderUpdateTime = Time.time; MeshCollider meshCollider = GetComponent(); if (meshCollider) { meshCollider.sharedMesh = null; meshCollider.sharedMesh = mesh; } } } } } } }