storytold/LiveScan3D
2
0
Fork 0
mirror of https://github.com/storytold/LiveScan3D.git synced 2026-10-09 00:09:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Skeleton data can now be seen in the client and in the live window. No skeleton merging/saving yet.

Browse Source
This commit is contained in:
marek
2015-11-05 18:56:53 +01:00
parent 6df7380372
commit 948eee769e
12 changed files with 584 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
LiveScanServer/KinectServer.cs
+3 -1
View File
@@ -277,10 +277,11 @@ namespace KinectServer
}
}
public void GetLatestFrame(List<List<byte>> lFramesRGB, List<List<Single>> lFramesVerts)
public void GetLatestFrame(List<List<byte>> lFramesRGB, List<List<Single>> lFramesVerts, List<List<Body>> lFramesBody)
{
lFramesRGB.Clear();
lFramesVerts.Clear();
lFramesBody.Clear();
lock (oFrameRequestLock)
{
@@ -317,6 +318,7 @@ namespace KinectServer
{
lFramesRGB.Add(new List<byte>(lClientSockets[i].lFrameRGB));
lFramesVerts.Add(new List<Single>(lClientSockets[i].lFrameVerts));
lFramesBody.Add(new List<Body>(lClientSockets[i].lBodies));
}
}
}
LiveScanServer/KinectSocket.cs
+55 -5
View File
@@ -19,6 +19,7 @@ using System.Text;
using System.Threading.Tasks;
using System.Net.Sockets;
namespace KinectServer
{
public delegate void SocketChangedHandler();
@@ -40,6 +41,7 @@ namespace KinectServer
public List<byte> lFrameRGB = new List<byte>();
public List<Single> lFrameVerts = new List<Single>();
public List<Body> lBodies = new List<Body>();
public event SocketChangedHandler eChanged;
@@ -150,6 +152,7 @@ namespace KinectServer
{
lFrameRGB.Clear();
lFrameVerts.Clear();
lBodies.Clear();
int nToRead;
byte[] buffer = new byte[1024];
@@ -160,7 +163,6 @@ namespace KinectServer
return;
}
oSocket.Receive(buffer, 4, SocketFlags.None);
//string result = System.Text.Encoding.UTF8.GetString(buffer);
@@ -185,16 +187,64 @@ namespace KinectServer
nAlreadyRead += oSocket.Receive(buffer, nAlreadyRead, nToRead - nAlreadyRead, SocketFlags.None);
}
int point_size = 3 + 3 * 4;
int n_vertices = nToRead / point_size;
//Receive depth and color data
int startIdx = 0;
int n_vertices = BitConverter.ToInt32(buffer, startIdx);
startIdx += 4;
for (int i = 0; i < n_vertices; i++)
{
for (int j = 0; j < 3; j++)
{
lFrameRGB.Add(buffer[i * point_size + j]);
lFrameVerts.Add(BitConverter.ToSingle(buffer, i * point_size + j * 4 + 3));
lFrameRGB.Add(buffer[startIdx++]);
}
for (int j = 0; j < 3; j++)
{
lFrameVerts.Add(BitConverter.ToSingle(buffer, startIdx));
startIdx += 4;
}
}
//Receive body data
int nBodies = BitConverter.ToInt32(buffer, startIdx);
startIdx += 4;
for (int i = 0; i < nBodies; i++)
{
Body tempBody = new Body();
tempBody.bTracked = BitConverter.ToBoolean(buffer, startIdx++);
int nJoints = BitConverter.ToInt32(buffer, startIdx);
startIdx += 4;
tempBody.lJoints = new List<Joint>(nJoints);
tempBody.lJointsInColorSpace = new List<Point2f>(nJoints);
for (int j = 0; j < nJoints; j++)
{
Joint tempJoint = new Joint();
Point2f tempPoint = new Point2f();
tempJoint.jointType = (JointType)BitConverter.ToInt32(buffer, startIdx);
startIdx += 4;
tempJoint.trackingState = (TrackingState)BitConverter.ToInt32(buffer, startIdx);
startIdx += 4;
tempJoint.position.X = BitConverter.ToSingle(buffer, startIdx);
startIdx += 4;
tempJoint.position.Y = BitConverter.ToSingle(buffer, startIdx);
startIdx += 4;
tempJoint.position.Z = BitConverter.ToSingle(buffer, startIdx);
startIdx += 4;
tempPoint.X = BitConverter.ToSingle(buffer, startIdx);
startIdx += 4;
tempPoint.Y = BitConverter.ToSingle(buffer, startIdx);
startIdx += 4;
tempBody.lJoints.Add(tempJoint);
tempBody.lJointsInColorSpace.Add(tempPoint);
}
lBodies.Add(tempBody);
}
}
LiveScanServer/MainWindowForm.cs
+13 -3
View File
@@ -31,6 +31,8 @@ using System.Net;
using System.Net.Sockets;
using System.Timers;
using System.Diagnostics;
namespace KinectServer
{
@@ -48,6 +50,8 @@ namespace KinectServer
List<byte> lAllColors = new List<byte>();
//Sensor poses from all of the sensors
List<AffineTransform> lAllCameraPoses = new List<AffineTransform>();
//Body data from all of the sensors
List<Body> lAllBodies = new List<Body>();
bool bServerRunning = false;
bool bRecording = false;
@@ -161,6 +165,7 @@ namespace KinectServer
oOpenGLWindow.vertices = lAllVertices;
oOpenGLWindow.colors = lAllColors;
oOpenGLWindow.cameraPoses = lAllCameraPoses;
oOpenGLWindow.bodies = lAllBodies;
oOpenGLWindow.settings = oSettings;
}
oOpenGLWindow.Run();
@@ -245,26 +250,30 @@ namespace KinectServer
{
List<List<byte>> lFramesRGB = new List<List<byte>>();
List<List<Single>> lFramesVerts = new List<List<Single>>();
List<List<Body>> lFramesBody = new List<List<Body>>();
BackgroundWorker worker = (BackgroundWorker)sender;
while (!worker.CancellationPending)
{
Thread.Sleep(1);
oServer.GetLatestFrame(lFramesRGB, lFramesVerts);
oServer.GetLatestFrame(lFramesRGB, lFramesVerts, lFramesBody);
//Update the vertex and color lists that are common between this class and the OpenGLWindow.
lock (lAllVertices)
{
lAllVertices.Clear();
lAllColors.Clear();
lAllBodies.Clear();
lAllCameraPoses.Clear();
for (int i = 0; i < lFramesRGB.Count; i++)
{
lAllVertices.AddRange(lFramesVerts[i]);
lAllColors.AddRange(lFramesRGB[i]);
lAllCameraPoses.Add(oServer.lCameraPoses[i]);
lAllBodies.AddRange(lFramesBody[i]);
}
lAllCameraPoses.AddRange(oServer.lCameraPoses);
}
//Notes the fact that a new frame was downloaded, this is used to estimate the FPS.
@@ -285,7 +294,8 @@ namespace KinectServer
//Download a frame from each client.
List<List<float>> lAllFrameVertices = new List<List<float>>();
List<List<byte>> lAllFrameColors = new List<List<byte>>();
oServer.GetLatestFrame(lAllFrameColors, lAllFrameVertices);
List<List<Body>> lAllFrameBody = new List<List<Body>>();
oServer.GetLatestFrame(lAllFrameColors, lAllFrameVertices, lAllFrameBody);
//Initialize containers for the poses.
List<float[]> Rs = new List<float[]>();
LiveScanServer/OpenGLWindow.cs
+67
View File
@@ -19,6 +19,8 @@ using System.Text;
using System.Threading.Tasks;
using System.Globalization;
using System.Diagnostics;
using OpenTK;
using OpenTK.Graphics;
using OpenTK.Input;
@@ -58,6 +60,7 @@ namespace KinectServer
public List<float> vertices = new List<float>();
public List<byte> colors = new List<byte>();
public List<AffineTransform> cameraPoses = new List<AffineTransform>();
public List<Body> bodies = new List<Body>();
public KinectSettings settings = new KinectSettings();
DateTime tFPSUpdateTimer = DateTime.Now;
@@ -298,6 +301,7 @@ namespace KinectServer
LineCount += settings.lMarkerPoses.Count * 3;
//cameras
LineCount += cameraPoses.Count * 3;
LineCount += 24 * bodies.Count;
VBO = new VertexC4ubV3f[PointCount + 2 * LineCount];
@@ -322,6 +326,7 @@ namespace KinectServer
{
iCurLineCount += AddCamera(PointCount + 2 * iCurLineCount, cameraPoses[i]);
}
iCurLineCount += AddBodies(PointCount + 2 * iCurLineCount);
}
}
}
@@ -544,6 +549,68 @@ namespace KinectServer
return nLinesBeingAdded;
}
private int AddBone(int bodyIdx, JointType jointType0, JointType jointType1, int startIdx)
{
Point3f joint0 = bodies[bodyIdx].lJoints[(int)jointType0].position;
Point3f joint1 = bodies[bodyIdx].lJoints[(int)jointType1].position;
AddLine(startIdx, joint0.X, joint0.Y, joint0.Z, joint1.X, joint1.Y, joint1.Z);
return 2;
}
private int AddBodies(int startIdx)
{
int nLinesToAdd = 24 * bodies.Count;
int nPointsToAdd = nLinesToAdd * 2;
for (int i = startIdx; i < startIdx + nPointsToAdd; i++)
{
VBO[i].R = 0;
VBO[i].G = 255;
VBO[i].B = 0;
VBO[i].A = 0;
}
int n = 0;
for (int bodyIdx = 0; bodyIdx < bodies.Count; bodyIdx++)
{
//Torso
n += AddBone(bodyIdx, JointType.JointType_Head, JointType.JointType_Neck, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_Neck, JointType.JointType_SpineShoulder, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_SpineShoulder, JointType.JointType_SpineMid, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_SpineMid, JointType.JointType_SpineBase, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_SpineShoulder, JointType.JointType_ShoulderRight, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_SpineShoulder, JointType.JointType_ShoulderLeft, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_SpineBase, JointType.JointType_HipRight, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_SpineBase, JointType.JointType_HipLeft, startIdx + n);
// Right Arm
n += AddBone(bodyIdx, JointType.JointType_ShoulderRight, JointType.JointType_ElbowRight, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_ElbowRight, JointType.JointType_WristRight, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_WristRight, JointType.JointType_HandRight, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_HandRight, JointType.JointType_HandTipRight, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_WristRight, JointType.JointType_ThumbRight, startIdx + n);
// Left Arm
n += AddBone(bodyIdx, JointType.JointType_ShoulderLeft, JointType.JointType_ElbowLeft, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_ElbowLeft, JointType.JointType_WristLeft, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_WristLeft, JointType.JointType_HandLeft, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_HandLeft, JointType.JointType_HandTipLeft, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_WristLeft, JointType.JointType_ThumbLeft, startIdx + n);
// Right Leg
n += AddBone(bodyIdx, JointType.JointType_HipRight, JointType.JointType_KneeRight, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_KneeRight, JointType.JointType_AnkleRight, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_AnkleRight, JointType.JointType_FootRight, startIdx + n);
// Left Leg
n += AddBone(bodyIdx, JointType.JointType_HipLeft, JointType.JointType_KneeLeft, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_KneeLeft, JointType.JointType_AnkleLeft, startIdx + n);
n += AddBone(bodyIdx, JointType.JointType_AnkleLeft, JointType.JointType_FootLeft, startIdx + n);
}
return nLinesToAdd;
}
private void AddLine(int startIdx, float x0, float y0, float z0,
float x1, float y1, float z1)
{
LiveScanServer/Utils.cs
+65
View File
@@ -13,9 +13,23 @@
// year={2015},
// }
using System;
using System.Collections.Generic;
namespace KinectServer
{
public struct Point2f
{
public float X;
public float Y;
}
public struct Point3f
{
public float X;
public float Y;
public float Z;
}
[Serializable]
public class AffineTransform
{
@@ -92,4 +106,55 @@ namespace KinectServer
private float[] r = new float[3];
}
public enum TrackingState
{
TrackingState_NotTracked = 0,
TrackingState_Inferred = 1,
TrackingState_Tracked = 2
}
public enum JointType
{
JointType_SpineBase = 0,
JointType_SpineMid = 1,
JointType_Neck = 2,
JointType_Head = 3,
JointType_ShoulderLeft = 4,
JointType_ElbowLeft = 5,
JointType_WristLeft = 6,
JointType_HandLeft = 7,
JointType_ShoulderRight = 8,
JointType_ElbowRight = 9,
JointType_WristRight = 10,
JointType_HandRight = 11,
JointType_HipLeft = 12,
JointType_KneeLeft = 13,
JointType_AnkleLeft = 14,
JointType_FootLeft = 15,
JointType_HipRight = 16,
JointType_KneeRight = 17,
JointType_AnkleRight = 18,
JointType_FootRight = 19,
JointType_SpineShoulder = 20,
JointType_HandTipLeft = 21,
JointType_ThumbLeft = 22,
JointType_HandTipRight = 23,
JointType_ThumbRight = 24,
JointType_Count = (JointType_ThumbRight + 1)
}
public struct Joint
{
public Point3f position;
public JointType jointType;
public TrackingState trackingState;
}
public struct Body
{
public bool bTracked;
public List<Joint> lJoints;
public List<Point2f> lJointsInColorSpace;
}
}