using UnityEngine;
using System.Collections;
using com.rfilkov.kinect;
namespace com.rfilkov.components
{
///
/// SceneMeshRenderer renders virtual mesh of the environment in the scene, as detected by the given sensor.
///
public class SceneMeshRenderer : MonoBehaviour
{
[Tooltip("Depth sensor index - 0 is the 1st one, 1 - the 2nd one, etc.")]
public int sensorIndex = 0;
[Tooltip("Resolution of the images used to generate the scene.")]
private DepthSensorBase.PointCloudResolution sourceImageResolution = DepthSensorBase.PointCloudResolution.DepthCameraResolution;
[Tooltip("Whether to show the mesh as point cloud, or as solid mesh.")]
public bool showAsPointCloud = true;
[Tooltip("Horizontal limit - minimum, in meters.")]
[Range(-5f, 5f)]
public float xMin = -2f;
[Tooltip("Horizontal limit - maximum, in meters.")]
[Range(-5f, 5f)]
public float xMax = 2f;
[Tooltip("Vertical limit - minimum, in meters.")]
[Range(-5f, 5f)]
public float yMin = -2f;
[Tooltip("Vertical limit - maximum, in meters.")]
[Range(-5f, 5f)]
public float yMax = 2f;
[Tooltip("Distance limit - minimum, in meters.")]
[Range(0.5f, 10f)]
public float zMin = 1f;
[Tooltip("Distance limit - maximum, in meters.")]
[Range(0.5f, 10f)]
public float zMax = 3f;
[Tooltip("Time interval between scene mesh updates, in seconds. 0 means no wait.")]
public float updateMeshInterval = 0f;
[Tooltip("Whether to include the detected players to the scene mesh or not.")]
private bool includePlayers = true;
[Tooltip("Whether to update the mesh, only when there are no players detected.")]
private bool updateWhenNoPlayers = false;
//[Tooltip("Whether to update the mesh collider as well, when the user mesh changes.")]
//public bool updateMeshCollider = false;
[Tooltip("Time interval between mesh-collider updates, in seconds. 0 means no mesh-collider updates.")]
public float updateColliderInterval = 0f;
// reference to object's mesh
private Mesh mesh = null;
// references to KM and data
private KinectManager kinectManager = null;
private KinectInterop.SensorData sensorData = null;
private DepthSensorBase sensorInt = null;
private Vector3 spaceScale = Vector3.zero;
// render textures
private RenderTexture colorTexture = null;
private RenderTexture colorTextureCopy = null;
private bool colorTextureCreated = false;
// times
private ulong lastSpaceCoordsTime = 0;
private float lastMeshUpdateTime = 0f;
private float lastColliderUpdateTime = 0f;
private int minDepth = 0;
private int maxDepth = 0;
// image parameters
private int imageWidth = 0;
private int imageHeight = 0;
// mesh parameters
private Vector3[] spaceTable = null;
private Vector3[] meshVertices = null;
private int[] meshIndices = null;
private byte[] meshVertUsed = null;
private bool bMeshInited = false;
void Start()
{
// get sensor data
kinectManager = KinectManager.Instance;
sensorData = (kinectManager != null && kinectManager.IsInitialized()) ? kinectManager.GetSensorData(sensorIndex) : null;
}
void OnDestroy()
{
if(bMeshInited)
{
// release the mesh-related resources
FinishMesh();
}
}
void LateUpdate()
{
if(mesh == null && sensorData != null && sensorData.depthCamIntr != null)
{
// init mesh and its related data
InitMesh();
}
if (bMeshInited)
{
// min & max depth
minDepth = Mathf.RoundToInt(zMin * 1000f);
maxDepth = Mathf.RoundToInt(zMax * 1000f);
// update the mesh
UpdateMesh();
}
}
// inits the mesh and related data
private void InitMesh()
{
// create mesh
mesh = new Mesh
{
name = "SceneMesh-Sensor" + sensorIndex,
indexFormat = UnityEngine.Rendering.IndexFormat.UInt32
};
MeshFilter meshFilter = GetComponent();
if(meshFilter != null)
{
meshFilter.mesh = mesh;
}
else
{
Debug.LogWarning("MeshFilter not found! You may not see the mesh on screen");
}
if (sensorData != null && sensorData.sensorInterface != null)
{
sensorInt = (DepthSensorBase)sensorData.sensorInterface;
Vector2Int imageRes = Vector2Int.zero;
if (sensorInt.pointCloudColorTexture == null)
{
sensorInt.pointCloudResolution = sourceImageResolution;
imageRes = sensorInt.GetPointCloudTexResolution(sensorData);
colorTexture = KinectInterop.CreateRenderTexture(colorTexture, imageRes.x, imageRes.y, RenderTextureFormat.ARGB32);
sensorInt.pointCloudColorTexture = colorTexture;
colorTextureCreated = true;
}
else
{
imageRes = sensorInt.GetPointCloudTexResolution(sensorData);
colorTexture = sensorInt.pointCloudColorTexture;
colorTextureCreated = false;
}
// create copy texture
colorTextureCopy = KinectInterop.CreateRenderTexture(colorTextureCopy, imageRes.x, imageRes.y, RenderTextureFormat.ARGB32);
// set the color texture
Renderer meshRenderer = GetComponent();
if (meshRenderer && meshRenderer.material && meshRenderer.material.mainTexture == null)
{
meshRenderer.material.mainTexture = colorTextureCopy; // sensorInt.pointCloudColorTexture;
//meshRenderer.material.SetTextureScale("_MainTex", kinectManager.GetColorImageScale(sensorIndex));
}
// image width & height
imageWidth = imageRes.x;
imageHeight = imageRes.y;
int pointCount = imageWidth * imageHeight;
// mesh arrays
meshVertices = new Vector3[pointCount];
meshIndices = new int[6 * pointCount]; // 2 triangles per vertex, last row and column excluded
meshVertUsed = new byte[pointCount];
spaceScale = kinectManager.GetSensorSpaceScale(sensorIndex);
// create space table
spaceTable = sensorInt.pointCloudResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution ?
sensorInt.GetDepthCameraSpaceTable(sensorData) : sensorInt.GetColorCameraSpaceTable(sensorData);
// init mesh uv array
//spaceTable = new Vector3[pointCount];
Vector2[] meshUv = new Vector2[pointCount];
for (int y = 0, i = 0; y < imageHeight; y++)
{
for (int x = 0; x < imageWidth; x++, i++)
{
Vector2 imagePos = new Vector2(x, y);
//spaceTable[i] = sensorInt.MapDepthPointToSpaceCoords(sensorData, imagePos, 1000);
//sourceImageResolution == DepthSensorBase.PointCloudResolution.ColorCameraResolution ?
//sensorInt.MapColorPointToSpaceCoords(sensorData, imagePos, 1000) :
//sensorInt.MapDepthPointToSpaceCoords(sensorData, imagePos, 1000);
meshUv[i] = new Vector2(imagePos.x / imageWidth, imagePos.y / imageHeight);
}
}
mesh.vertices = meshVertices;
mesh.SetIndices(meshIndices, MeshTopology.Triangles, 0);
mesh.uv = meshUv;
if(showAsPointCloud)
{
meshIndices = new int[pointCount];
for (int i = 0; i < pointCount; i++)
meshIndices[i] = i;
mesh.SetIndices(meshIndices, MeshTopology.Points, 0);
}
bMeshInited = true;
}
}
// releases mesh-related resources
private void FinishMesh()
{
if (sensorInt)
{
sensorInt.pointCloudColorTexture = null;
}
if (colorTexture && colorTextureCreated)
{
colorTexture.Release();
colorTexture = null;
}
if (colorTextureCopy)
{
colorTextureCopy.Release();
colorTextureCopy = null;
}
spaceTable = null;
meshVertices = null;
meshIndices = null;
meshVertUsed = null;
bMeshInited = false;
}
// updates the mesh according to current depth frame
private void UpdateMesh()
{
if (bMeshInited && sensorData.depthImage != null && lastSpaceCoordsTime != sensorData.lastDepthFrameTime)
{
// check for players
bool bHavePlayers = kinectManager.GetUsersCount() != 0;
if ((Time.time - lastMeshUpdateTime) >= updateMeshInterval && (!updateWhenNoPlayers || !bHavePlayers))
{
lastSpaceCoordsTime = sensorData.lastDepthFrameTime;
lastMeshUpdateTime = Time.time;
int imageWidth1 = imageWidth - 1, imageHeight1 = imageHeight - 1;
const int maxDistMm = 200; // max distance between vertices in a triangle, in mm
// copy the texture
Graphics.CopyTexture(colorTexture, colorTextureCopy);
for (int y = 0, di = 0, ti = 0; y < imageHeight; y++)
{
for (int x = 0; x < imageWidth; x++, di++, ti += 6)
{
ushort depth = sensorData.depthImage[di];
bool isValidBodyPixel = !includePlayers && bHavePlayers ? sensorData.bodyIndexImage[di] == 255 : true;
bool bVertexSet = false;
if (depth >= minDepth && depth <= maxDepth && isValidBodyPixel)
{
float fDepth = (float)depth * 0.001f;
Vector3 vVertex = new Vector3(spaceTable[di].x * fDepth * spaceScale.x, spaceTable[di].y * fDepth * spaceScale.y, fDepth);
bool bUsedVertex = !showAsPointCloud && meshVertUsed[di] != 0;
if (bUsedVertex || (vVertex.x >= xMin && vVertex.x <= xMax && vVertex.y >= yMin && vVertex.y <= yMax))
{
meshVertices[di] = vVertex;
if (!showAsPointCloud)
{
meshVertUsed[di] = 1;
if (x < imageWidth1 && y < imageHeight1)
{
int tl = di; ushort tld = depth;
int tr = tl + 1; ushort trd = sensorData.depthImage[tr];
int bl = tl + imageWidth; ushort bld = sensorData.depthImage[bl];
int br = bl + 1; ushort brd = sensorData.depthImage[br];
float fBrDepth = (float)brd * 0.001f;
Vector3 vVertex2 = new Vector3(spaceTable[br].x * fBrDepth * spaceScale.x, spaceTable[di].y * fBrDepth * spaceScale.y, fBrDepth);
// 1st triangle
if (tld >= minDepth && tld <= maxDepth &&
trd >= minDepth && trd <= maxDepth &&
bld >= minDepth && bld <= maxDepth &&
vVertex2.x >= xMin && vVertex2.x <= xMax && vVertex2.y >= yMin && vVertex2.y <= yMax &&
Mathf.Abs(trd - tld) < maxDistMm && Mathf.Abs(bld - tld) < maxDistMm)
{
meshIndices[ti] = bl;
meshIndices[ti + 1] = tr;
meshIndices[ti + 2] = tl;
meshVertUsed[trd] = meshVertUsed[bld] = 1;
}
else
{
meshIndices[ti] = meshIndices[ti + 1] = meshIndices[ti + 2] = 0;
meshVertUsed[trd] = meshVertUsed[bld] = 0;
}
// 2nd triangle
if (bld >= minDepth && bld <= maxDepth &&
trd >= minDepth && trd <= maxDepth &&
brd >= minDepth && brd <= maxDepth &&
vVertex2.x >= xMin && vVertex2.x <= xMax && vVertex2.y >= yMin && vVertex2.y <= yMax &&
Mathf.Abs(trd - bld) < maxDistMm && Mathf.Abs(brd - bld) < maxDistMm)
{
meshIndices[ti + 3] = br;
meshIndices[ti + 4] = tr;
meshIndices[ti + 5] = bl;
meshVertUsed[bld] = meshVertUsed[trd] = meshVertUsed[brd] = 1;
}
else
{
meshIndices[ti + 3] = meshIndices[ti + 4] = meshIndices[ti + 5] = 0;
meshVertUsed[bld] = meshVertUsed[trd] = meshVertUsed[brd] = 0;
}
}
else
{
meshIndices[ti] = meshIndices[ti + 1] = meshIndices[ti + 2] = 0;
meshIndices[ti + 3] = meshIndices[ti + 4] = meshIndices[ti + 5] = 0;
}
}
bVertexSet = true;
}
}
if (!bVertexSet)
{
meshVertices[di] = Vector3.zero;
if (!showAsPointCloud)
{
meshIndices[ti] = meshIndices[ti + 1] = meshIndices[ti + 2] = 0;
meshIndices[ti + 3] = meshIndices[ti + 4] = meshIndices[ti + 5] = 0;
}
}
}
}
mesh.vertices = meshVertices;
if(!showAsPointCloud)
{
mesh.SetIndices(meshIndices, MeshTopology.Triangles, 0);
}
mesh.RecalculateBounds();
if (updateColliderInterval > 0 && (Time.time - lastColliderUpdateTime) >= updateColliderInterval)
{
lastColliderUpdateTime = Time.time;
MeshCollider meshCollider = GetComponent();
if (meshCollider)
{
meshCollider.sharedMesh = null;
meshCollider.sharedMesh = mesh;
}
}
}
}
}
}
}