kinect2osc.pde

gistfile1.txt

import oscP5.*;
import netP5.*;
import SimpleOpenNI.*;

OscP5 osc;
NetAddress osc_send_addr;
NetAddress tuio_send_addr;

SimpleOpenNI      context;
XnVSessionManager sessionManager;
XnVPushDetector   pushDetector;
XnVSwipeDetector  swipeDetector;
XnVCircleDetector circleDetector;
HandTracker       handTracker;

float last_circ_times;

void setup() {
  //OSC SETUP
  osc = new OscP5(this, 12221);
  tuio_send_addr = new NetAddress("127.0.0.1", 3333);
  osc_send_addr = new NetAddress("127.0.0.1", 12222);

  //KINECT CONTEXT
  context = new SimpleOpenNI(this);
  context.setMirror(true);
  context.enableDepth();
  context.enableGesture();
  context.enableHands();
  context.enableScene();
  context.enableUser(SimpleOpenNI.SKEL_PROFILE_ALL);

  //PUSH
  pushDetector = new XnVPushDetector();    
  pushDetector.RegisterPush(this);

  circleDetector = new XnVCircleDetector();
  circleDetector.RegisterCircle(this);
  last_circ_times = 0.0;

  //SWIPE
  swipeDetector = new XnVSwipeDetector();
  swipeDetector.SetSteadyDuration((long)50.0);
  swipeDetector.SetMotionSpeedThreshold(.45);
  swipeDetector.SetSteadyMaxStdDev(0.2);
  swipeDetector.RegisterSwipeLeft(this);
  swipeDetector.RegisterSwipeRight(this);

  //HAND TRACKER
  handTracker = new HandTracker();

  // GESTURE AND HAND TRACKER 
  sessionManager = context.createSessionManager("Wave", "RaiseHand");
  sessionManager.SetQuickRefocusTimeout(800); //default 15000
  println("SESSION TIMEOUT:" + sessionManager.GetQuickRefocusTimeout());
  sessionManager.AddListener(pushDetector);
  sessionManager.AddListener(swipeDetector);
  sessionManager.AddListener(handTracker);
sessionManager.AddListener(circleDetector);

  //GRAPHICS         
  size(context.depthWidth(), context.depthHeight()); 
  smooth();
}

void draw() {
  background(200, 0, 0);
  context.update();
  //println(context.getNumberOfUsers());
  context.update(sessionManager);
  image(context.sceneImage(), 0, 0);
  handTracker.draw();
}



/////////////////////////////////////////////////////////////////////////////////////////////////////
// OSC SENDING



/////////////////////////////////////////////////////////////////////////////////////////////////////
// TUIO SENDING

//TODO: add support for multiple cursors
int session_uid = 0;
int frame_uid = 0;

void tuio2D_start(int id, float x, float y, float dx, float dy, float accel) {
  session_uid = session_uid +1;
  frame_uid = frame_uid +1;

  println("START TUIO"+x+" "+y);
  OscMessage aliveMessage = new OscMessage("/tuio/2Dcur"); 
  aliveMessage.add("alive"); 
  aliveMessage.add(session_uid); 

  OscMessage setMessage = new OscMessage("/tuio/2Dcur"); 
  setMessage.add("set"); 
  setMessage.add(session_uid); 
  setMessage.add(x); 
  setMessage.add(y); 
  setMessage.add(dx); 
  setMessage.add(dy); 
  setMessage.add(accel); 

  OscMessage frameMessage = new OscMessage("/tuio/2Dcur"); 
  frameMessage.add("frame");
  frameMessage.add(frame_uid);

  OscBundle blobsBundle = new OscBundle();
  blobsBundle.add(aliveMessage);
  blobsBundle.add(setMessage);
  blobsBundle.add(frameMessage);
  osc.send(blobsBundle, tuio_send_addr);
}

void tuio2D_update(int id, float x, float y, float dx, float dy, float accel) {
  frame_uid = frame_uid +1;
  
  OscMessage aliveMessage = new OscMessage("/tuio/2Dcur"); 
  aliveMessage.add("alive"); 
  aliveMessage.add(session_uid); 
  print("ALIVE: ");
  print(session_uid);
  print("\n");

  OscMessage setMessage = new OscMessage("/tuio/2Dcur"); 
  setMessage.add("set"); 
  setMessage.add(session_uid); 
  setMessage.add(x); 
  setMessage.add(y); 
  setMessage.add(dx); 
  setMessage.add(dy); 
  setMessage.add(accel); 

  OscMessage frameMessage = new OscMessage("/tuio/2Dcur"); 
  frameMessage.add("frame");
  frameMessage.add(frame_uid);

  OscBundle blobsBundle = new OscBundle();
  blobsBundle.add(aliveMessage);
  blobsBundle.add(setMessage);
  blobsBundle.add(frameMessage);
  osc.send(blobsBundle, tuio_send_addr);
}

void tuio2D_stop(int id) {
  print("STOP: ");
  print(session_uid);
  print("\n");

 
  frame_uid = frame_uid +1;
  OscMessage aliveMessage = new OscMessage("/tuio/2Dcur"); 
  aliveMessage.add("alive"); 

  OscMessage frameMessage = new OscMessage("/tuio/2Dcur"); 
  frameMessage.add("frame");
  frameMessage.add(frame_uid);

  OscBundle blobsBundle = new OscBundle();
  blobsBundle.add(aliveMessage);
  blobsBundle.add(frameMessage);
  osc.send(blobsBundle, tuio_send_addr);
}



/////////////////////////////////////////////////////////////////////////////////////////////////////
// user callbacks
void onNewUser(int userId) {
  println("User ENTER - userId: " + userId);
  OscMessage msg = new OscMessage("/kinect/user/enter");
  msg.add(userId);
  osc.send(msg, osc_send_addr);
}

void onLostUser(int userId) {
  println("User EXIT - userId: " + userId);
  OscMessage msg = new OscMessage("/kinect/user/exit");
  msg.add(userId);
  osc.send(msg, osc_send_addr);
}


/////////////////////////////////////////////////////////////////////////////////////////////////////
// session callbacks

void onStartSession(PVector pos)
{
  OscMessage msg = new OscMessage("/kinect/session/start");
  msg.add(pos.x);
  msg.add(pos.y);
  msg.add(pos.z);
  osc.send(msg, osc_send_addr);
  println("onStartSession: " + pos);
}

void onEndSession()
{
  OscMessage msg = new OscMessage("/kinect/session/end");
  osc.send(msg, osc_send_addr);
  println("onEndSession: ");
}

void onFocusSession(String strFocus, PVector pos, float progress)
{
  OscMessage msg = new OscMessage("/kinect/session/progress");
  msg.add(strFocus);
  msg.add(pos.x);
  msg.add(pos.y);
  msg.add(pos.z);
  msg.add(progress);
  osc.send(msg, osc_send_addr);
  println("onFocusSession: focus=" + strFocus + ",pos=" + pos + ",progress=" + progress);
}


/////////////////////////////////////////////////////////////////////////////////////////////////////
// gesture callbacks

void onPush(float velocity, float angle) {
  OscMessage msg = new OscMessage("/kinect/gesture/push");
  msg.add(velocity);
  msg.add(angle);
  osc.send(msg, osc_send_addr);
  println("PUSH: "+ velocity+"  "+angle);
}

void onSwipeLeft( float velocity, float angle) {
  OscMessage msg = new OscMessage("/kinect/gesture/swipe");
  msg.add("left");
  msg.add(velocity);
  msg.add(angle);
  osc.send(msg, osc_send_addr);
  println("SWIPE LEFT: "+ velocity+"  "+angle);
}

void onSwipeRight(float velocity, float angle) {
  OscMessage msg = new OscMessage("/kinect/gesture/swipe");
  msg.add("right");
  msg.add(velocity);
  msg.add(angle);
  osc.send(msg, osc_send_addr);
  println("SWIPE RIGHT: "+ velocity+"  "+angle);
}

void onCircle(float fTimes,boolean bConfident,XnVCircle circle)
{
  if (abs(last_circ_times - fTimes) < 0.25)
    return;
  
  last_circ_times = fTimes;
  int conf = 0;
  if (bConfident)
    conf =1;
    
  OscMessage msg = new OscMessage("/kinect/gesture/circle");
  msg.add(fTimes);
  msg.add(conf);
  osc.send(msg, osc_send_addr);
  println("CIRCLE: "+ fTimes+"  "+bConfident);


}


/////////////////////////////////////////////////////////////////////////////////////////////////////
// HandTracker

class HandTracker extends XnVPointControl
{
  HashMap    _pointLists;
  int        _maxPoints;
  color[]    _colorList = { 
    color(255, 0, 0), color(0, 255, 0), color(0, 0, 255), color(255, 255, 0)
  };

  public HandTracker()
  {
    _maxPoints = 20;
    _pointLists = new HashMap();
  }

  public void OnPointCreate(XnVHandPointContext cxt) {

    PVector screenPos= new PVector();
    PVector pos = new PVector(cxt.getPtPosition().getX(), cxt.getPtPosition().getY(), cxt.getPtPosition().getZ());
    context.convertRealWorldToProjective(pos, screenPos);
    addPoint(cxt.getNID(), pos);
    tuio2D_start((int)cxt.getNID(), screenPos.x*1.0/width, screenPos.y*1.0/height, 0, 0, pos.z);
  }

  public void OnPointUpdate(XnVHandPointContext cxt) {
    PVector screenPos= new PVector();
    PVector pos = new PVector(cxt.getPtPosition().getX(), cxt.getPtPosition().getY(), cxt.getPtPosition().getZ());
    context.convertRealWorldToProjective(pos, screenPos);
    addPoint(cxt.getNID(), pos);
    tuio2D_update((int)cxt.getNID(), screenPos.x*1.0/width, screenPos.y*1.0/height, 0, 0, pos.z);
  }

  public void OnPointDestroy(long nID) {
    if (_pointLists.containsKey(nID)){
      _pointLists.remove(nID);
      tuio2D_stop((int)nID);
    }
  }

  public ArrayList getPointList(long handId) {
    ArrayList curList;
    if (_pointLists.containsKey(handId))
      curList = (ArrayList)_pointLists.get(handId);
    else
    {
      curList = new ArrayList(_maxPoints);
      _pointLists.put(handId, curList);
    }
    return curList;
  }

  public void addPoint(long handId, PVector handPoint) {
    ArrayList curList = getPointList(handId);

    curList.add(0, handPoint);      
    if (curList.size() > _maxPoints)
      curList.remove(curList.size() - 1);
  }

  public void draw() {
    if (_pointLists.size() <= 0)
      return;

    pushStyle();
    noFill();

    PVector vec;
    PVector firstVec;
    PVector screenPos = new PVector();
    int colorIndex=0;

    // draw the hand lists
    Iterator<Map.Entry> itrList = _pointLists.entrySet().iterator();
    while (itrList.hasNext ()) 
    {
      strokeWeight(2);
      stroke(_colorList[colorIndex % (_colorList.length - 1)]);

      ArrayList curList = (ArrayList)itrList.next().getValue();     

      // draw line
      firstVec = null;
      Iterator<PVector> itr = curList.iterator();
      beginShape();
      while (itr.hasNext ()) 
      {
        vec = itr.next();
        if (firstVec == null)
          firstVec = vec;
        // calc the screen pos
        context.convertRealWorldToProjective(vec, screenPos);
        vertex(screenPos.x, screenPos.y);
      } 
      endShape();   

      // draw current pos of the hand
      if (firstVec != null)
      {
        strokeWeight(8);
        context.convertRealWorldToProjective(firstVec, screenPos);
        point(screenPos.x, screenPos.y);
      }
      colorIndex++;
    }

    popStyle();
  }
}