1. EEC-693/793 Applied Computer Vision with Depth Cameras
Lecture 17
Wenbing Zhao

2. Outline
Skeleton logging
Replaying in Unity with logged skeleton data

3. Skeleton Joint Data Logging
You can add logging for skeleton joint data into any of your existing project with skeleton tracking enabled
Make sure you add the System.IO into your namespace
We will use StreamWriter class for logging to a comma separated value file (.csv). Add the following member variable to your MainWindow class:
using System.IO;
StreamWriter sw = File.CreateText("kinect.csv"); 3

4. Skeleton Joint Data Logging
Add captions for columns. Add the following line in WindowLoaded()
Log per frame information, including
Timestamp
Frame number
Floor clip plane
sw.WriteLine("FrameNo,timestsamp,MC-X,MC-Y,MC-Z,MC-D,HipCenter-X,HipCenter-Y,HipCenter-Z,HipCenter-D,Spine-X,Spine-Y,Spine-Z,Spine-D,ShoulderCenter-X,ShoulderCenter-Y,ShoulderCenter-Z,ShoulderCenter-D,Head-X,Head-Y,Head-Z,Head-D,ShoulderLeft-X,ShoulderLeft-Y,ShoulderLeft-Z,ShoulderLeft-D,ElbowLeft-X,ElbowLeft-Y,ElbowLeft-Z,ElbowLeft-D,WristLeft-X,WristLeft-Y,WristLeft-Z,WristLeft-D,HandLeft-X,HandLeft-Y,HandLeft-Z,HandLeft-D,ShoulderRight-X,ShoulderRight-Y,ShoulderRight-Z,ShoulderRight-D,ElbowRight-X,ElbowRight-Y,ElbowRight-Z,ElbowRight-D,WristRight-X,WristRight-Y,WristRight-Z,WristRight-D,HandRight-X,HandRight-Y,HandRight-Z,HandRight-D,HipLeft-X,HipLeft-Y,HipLeft-Z,HipLeft-D,KneeLeft-X,KneeLeft-Y,KneeLeft-Z,KneeLeft-D,AnkleLeft-X,AnkleLeft-Y,AnkleLeft-Z,AnkleLeft-D,FootLeft-X,FootLeft-Y,FootLeft-Z,FootLeft-D,HipRight-X,HipRight-Y,HipRight-Z,HipRight-D,KneeRight-X,KneeRigt-Y,KneeRight-Z,KneeRight-D,AnkleRight-X,AnkleRight-Y,AnkleRight-Z,AnkleRight-D,FootRight-X,FootRight-Y,FootRight-Z,FootRight-D"); 4

5. Per Frame Information
void skeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e) {
canvas1.Children.Clear();
long timestamp = -1;
int frameno = -1;
float A = 0;
float B = 0;
float C = 0;
float D = 0;
using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame())
// check for frame drop.
if (skeletonFrame == null)
return; } 5

6. Per Frame Info timestamp = skeletonFrame.Timestamp;
frameno = skeletonFrame.FrameNumber;
A = skeletonFrame.FloorClipPlane.Item1;
B = skeletonFrame.FloorClipPlane.Item2;
C = skeletonFrame.FloorClipPlane.Item3;
D = skeletonFrame.FloorClipPlane.Item4;
// copy the frame data in to the collection
skeletonFrame.CopySkeletonDataTo(totalSkeleton);
// get the first Tracked skeleton
skeleton = (from trackskeleton in totalSkeleton
where trackskeleton.TrackingState == SkeletonTrackingState.Tracked
select trackskeleton).FirstOrDefault();
// if the first skeleton returns null
if (skeleton == null)
return; 6

7. Log Per Frame Info
if (skeleton != null && this.currentSkeletonID != skeleton.TrackingId) {
this.currentSkeletonID = skeleton.TrackingId;
int totalTrackedJoints = skeleton.Joints.Where(item => item.TrackingState == JointTrackingState.Tracked).Count();
string TrackedTime = DateTime.Now.ToString("hh:mm:ss");
string status = "Skeleton Id: " + this.currentSkeletonID + ", total tracked joints: " + totalTrackedJoints + ", TrackTime: " + TrackedTime+"\n";
this.textBlock1.Text += status; }
DrawSkeleton(skeleton);
sw.Write(frameno+”,”);
sw.Write(timestamp+","); 7

8. Log Joint Info float xc = skeleton.Position.X;
float yc = skeleton.Position.Y;
float zc = skeleton.Position.Z;
float cmd = A * xc + B * yc + C * zc + D;
sw.Write(xc + "," + yc + "," + zc + "," + cmd + ",");
for (int i = 0; i < 20; i++) {
float x = skeleton.Joints[(JointType)i].Position.X;
float y = skeleton.Joints[(JointType)i].Position.Y;
float z = skeleton.Joints[(JointType)i].Position.Z;
float dist = A * x + B * y + C * z + D;
sw.Write(x + "," + y + "," + z + "," + dist + ","); }
sw.WriteLine(""); 8

9. A Sample Log
If you have Excel in your computer, you can double click the csv file to open it 9

10. Replay Logged Data in Unity
Create a new Unity project. It is based on the KinectPointMan example as part of the ZDK download
Add the floor the same way as before
Create an empty game object, name it KinectPointMan
Under the KinectPointMan, add 20 3D spheres, and name them properly using the 20 Kinect joints
Create a new C# script, I named it BackScript.cs
Attach the script to the KinectPointMan
Connect the 20 spheres in the scene with 20 joints listed in BackScript.cs
Save the scene as KinectPlayer 10

11. Replay Logged Data in Unity11

12. Connect the 20 Joints to the Script12

13. BackScript.cs
using UnityEngine; using System.Collections; using System.Xml; using System.ComponentModel; using System.Collections.Generic; using System.Linq; using System.Text; using System; using System.IO; public class BackScript : MonoBehaviour {     public GameObject Hip_Center;     public GameObject Spine;     public GameObject Shoulder_Center;     public GameObject Head;     public GameObject Shoulder_Left;     public GameObject Elbow_Left;     public GameObject Wrist_Left;     public GameObject Hand_Left; 13

14. BackScript.cs
    public GameObject Shoulder_Right;     public GameObject Elbow_Right;     public GameObject Wrist_Right;     public GameObject Hand_Right;     public GameObject Hip_Left;     public GameObject Knee_Left;     public GameObject Ankle_Left;     public GameObject Foot_Left;     public GameObject Hip_Right;     public GameObject Knee_Right;     public GameObject Ankle_Right;     public GameObject Foot_Right;          private GameObject[] _bones; //internal handle for the bones of the model   
    GameObject hip2spine;     GameObject spine2shoulder;     GameObject shoulder2head;     GameObject leftShoulder;     GameObject rightShoulder;     GameObject leftUpperArm; 14

15. BackScript.cs
    GameObject leftArm;     GameObject leftHand;     GameObject rightUpperArm;     GameObject rightArm;     GameObject rightHand;     GameObject leftHip;     GameObject rightHip;     GameObject leftUpperLeg;     GameObject leftLeg;     GameObject leftFoot;     GameObject rightUpperLeg;     GameObject rightLeg;     GameObject rightFoot;
    public GUIText statusDisplay;     List<string> exitems = new List<string>();     List<string> exfileitems = new List<string>();     List<List<int>> exjointsitems = new List<List<int>>();     List<float[][]> exdata = new List<float[][]>(); 15

16. BackScript.cs
    private int guideCounter = 0;     bool skipframe = false;     public Vector3 Scale = new Vector3(0.001f, 0.001f, 0.001f);
    float lastTs;     private Vector3 rootPosition = new Vector3();     // Use this for initialization     void Start () {         //store bones in a list for easier access         _bones = new GameObject[] {Hip_Center, Spine, Shoulder_Center, Head,             Shoulder_Left, Elbow_Left, Wrist_Left, Hand_Left,             Shoulder_Right, Elbow_Right, Wrist_Right, Hand_Right,             Hip_Left, Knee_Left, Ankle_Left, Foot_Left,             Hip_Right, Knee_Right, Ankle_Right, Foot_Right};         connectBones();     }
    void Update () {         if (exdata.Count == 0)             return;     } 16

17. BackScript.cs
    void OnGUI()     {         if (GUI.Button (new Rect (10, 70, 150, 30), "Load Kinect v1 Data")) {             loadConfig(true);             Debug.Log ("Kinect v1 data is loaded");         }
        if (Event.current.Equals(Event.KeyboardEvent("escape"))) {             print("Quitting");             Application.Quit();         }         if (Event.current.Equals(Event.KeyboardEvent("p"))) {             print ("Previous Frame");             if(this.guideCounter > 2)                 this.guideCounter = this.guideCounter - 2;             DisplayNextFrame();         }         if (Event.current.Equals(Event.KeyboardEvent("n"))) {             print ("Next Frame");             DisplayNextFrame();         }         if (Event.current.Equals(Event.KeyboardEvent("s"))) {             string filename = GetScreenshotFilename();             print("Writing screenshot to " + filename);             Application.CaptureScreenshot(filename);         }     } 17

18. BackScript.cs
    private void loadConfig(bool kinectv1data)     {                         XmlDocument xmlDoc = new XmlDocument ();         xmlDoc.Load ("test.xml");              XmlNodeList fileName = xmlDoc.GetElementsByTagName ("GuideFile");         if (1 != fileName.Count) {                 Debug.Log ("expecting exactly one guidefile, got: " + fileName.Count);                 return;         }         XmlNodeList innerlist = fileName [0].ChildNodes;         if (1 != innerlist.Count) {                 Debug.Log ("guidefile should have only one child, got: " + innerlist.Count);         }         String fn = innerlist [0].InnerText;         float[][] guideData = getGuideData (fn);         exdata.Add (guideData);     } 18

19. BackScript.cs 20 joints + misc. Time Jagged Array: allSamples
    private float[][] getGuideData(String filename)     {         String[] dataStrings = null;                 if (File.Exists(filename))         {             dataStrings = File.ReadAllLines(filename);             Debug.Log("Replay: done loading data");         }         else         {             Debug.Log("file not found: "+filename);             return null;         }                  // the first row is the caption, skip it, hence the -1         int size = dataStrings.Length-1;         float[][] allSamples = new float[size][];
20 joints + misc.
Frame
Time
Jagged Array: allSamples 19

20. BackScript.cs
        // the first row is the caption, skip it         for (int i = 0; i < allSamples.Length; i++)         {             String[] row = dataStrings[i + 1].Split(',');             int rowlen = 1+20*3; // 1 column for ts, 20 sets of joints (x,y,z)             allSamples[i] = new float[rowlen];             int k = 0;
           for(int j=0; j<row.Length; j++) {             // column 0 is for frame number              if(j==0)                  continue;              // column 2,3,4,5 are for center of mass              if(j >= 2 && j <=5)                  continue;             // for each joint, the 4th column is for vertical distance, skip it too             if(j > 6 && ((j-5) % 4)==0)                  continue;             allSamples[i][k++]=(float)Convert.ToDouble(row[j]);             if(k == 61)                  break;
}     }
return allSamples; } 20

21. BackScript.cs
    string GetScreenshotFilename()     {         System.IO.Directory.CreateDirectory("Screenshots");         int i=1;         while (System.IO.File.Exists(System.IO.Path.Combine("Screenshots", "Screenshot" + i + ".png"))) {             i++;         }         return System.IO.Path.Combine("Screenshots", "Screenshot" + i + ".png");     } 21

22. BackScript.cs: DisplayNextFrame()
    void DisplayNextFrame() {         float[][] guideData = exdata[0];         float[] frame;         if(guideCounter < guideData.Length)          {             frame = guideData[guideCounter];             if(null == frame)  {  return; }
            // first column is frame number; second column is timestamp;  first 3 are for rootposition             Vector3 jpt = new Vector3();             int jointCount = 20;             for(int i=0; i<jointCount; i++)             {                 jpt.x = 1200 - frame[1+3*i]*1000;                 jpt.y = frame[2+3*i]* ;                 jpt.z = frame[3+3*i]*1000;
                Debug.Log ("x="+jpt.x+",y="+jpt.y+",z="+jpt.z);                                      Vector3 dest = Vector3.Scale(jpt, Scale); 
                _bones[i].transform.position = dest;             }                 updateBones();
guideCounter++;         }
if(guideCounter >= guideData.Length)
guideCounter = 0;     } 22

23. BackScript.cs: connectBones()
    void connectBones()     {         this.hip2spine = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         this.spine2shoulder = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         shoulder2head = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         leftShoulder = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         rightShoulder = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         leftUpperArm = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         leftArm = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         leftHand = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         rightUpperArm = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         rightArm = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         rightHand = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         leftHip = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         rightHip = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         leftUpperLeg = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         leftLeg = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         leftFoot = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         rightUpperLeg = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         rightLeg = GameObject.CreatePrimitive(PrimitiveType.Cylinder);         rightFoot = GameObject.CreatePrimitive(PrimitiveType.Cylinder); 23

24. BackScript.cs: connectBones()
        hip2spine.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);         spine2shoulder.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);         shoulder2head.transform.localScale = new Vector3(0.05f, 0.1f, 0.05f);         leftShoulder.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);         rightShoulder.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);         leftUpperArm.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);         leftArm.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);         leftHand.transform.localScale = new Vector3(0.05f, 0.1f, 0.05f);         rightUpperArm.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);         rightArm.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);         rightHand.transform.localScale = new Vector3(0.05f, 0.1f, 0.05f);         leftHip.transform.localScale = new Vector3(0.05f, 0.1f, 0.05f);         rightHip.transform.localScale = new Vector3(0.05f, 0.1f, 0.05f);         leftUpperLeg.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);         leftLeg.transform.localScale = new Vector3(0.05f, 0.25f, 0.05f);         leftFoot.transform.localScale = new Vector3(0.05f, 0.15f, 0.05f);         rightUpperLeg.transform.localScale = new Vector3(0.15f, 0.25f, 0.15f);         rightLeg.transform.localScale = new Vector3(0.15f, 0.25f, 0.15f);         rightFoot.transform.localScale = new Vector3(0.1f, 0.1f, 0.1f);     }      24

25. BackScript.cs
    void updateBones()     {         updateOneBone(hip2spine, Hip_Center, Spine);         updateOneBone(spine2shoulder, Spine, Shoulder_Center);         updateOneBone(shoulder2head, Shoulder_Center, Head);         updateOneBone(leftShoulder, Shoulder_Center, Shoulder_Left);         updateOneBone(rightShoulder, Shoulder_Center, Shoulder_Right);         updateOneBone(leftUpperArm, Shoulder_Left, Elbow_Left);         updateOneBone(leftArm, Elbow_Left, Wrist_Left);         updateOneBone(leftHand, Wrist_Left, Hand_Left);         updateOneBone(rightUpperArm, Shoulder_Right, Elbow_Right);         updateOneBone(rightArm, Elbow_Right, Wrist_Right);         updateOneBone(rightHand, Wrist_Right, Hand_Right);         updateOneBone(leftHip, Hip_Center, Hip_Left);         updateOneBone(rightHip, Hip_Center, Hip_Right);         updateOneBone(leftUpperLeg, Hip_Left, Knee_Left);         updateOneBone(leftLeg, Knee_Left, Ankle_Left);         updateOneBone(leftFoot, Ankle_Left, Foot_Left);         updateOneBone(rightUpperLeg, Hip_Right, Knee_Right);         updateOneBone(rightLeg, Knee_Right, Ankle_Right);         updateOneBone(rightFoot, Ankle_Right, Foot_Right);
    }   25

26. BackScript.cs
    void updateOneBone(GameObject bone, GameObject start, GameObject end)     {         Vector3 v3End = end.transform.position;         Vector3 v3Start = start.transform.position;         bone.transform.position = (v3End-v3Start)/2.0f + v3Start;
        bone.transform.rotation = Quaternion.FromToRotation(Vector3.up, v3End-v3Start);             }      26

27. To Run The Replay Unity App
Make sure in the Unity app folder you have the following two files: test.xml and kinect.csv
You need to locate kinect.csv you saved previously for the logging part and copy it here
Kinect.csv may have to be cleaned up a little before feeding to the Unity app
Make sure the last row is complete. Remove the row if incomplete
test.xml content:
<?xml version="1.0" encoding="utf-8"?>
<ETatHome>
<GuideFile>kinect.csv</GuideFile>
</ETatHome> 27

28. Challenge Tasks
For logging, add a start and stop button for starting and stopping the logging
For replaying, also add a start and stop button, and a target-frame-rate button, for automated replaying (instead of manually display frame by frame). Tweak parameters so that different frame rates can be used: default, 30 frames/second, 20, and 10, etc.
For replaying, add display for joint angle for a joint that you choose 28