storytold/storyteller-ml
2
0
Fork 0
mirror of https://github.com/storytold/storyteller-ml.git synced 2026-10-18 07:41:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add MocapNET

Browse Source
This commit is contained in:
Justin John
2023-12-15 04:22:41 +05:30
parent 8434144ded
commit 65e8934d5e
336 changed files with 100584 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
animation/MocapNET/src/JointEstimator2D/CMakeLists.txt
+45
View File
@@ -0,0 +1,45 @@
project( JointEstimator2D )
cmake_minimum_required(VERSION 3.5)
set_property(GLOBAL PROPERTY USE_FOLDERS ON)
#set(CMAKE_CXX_STANDARD 11)
include_directories(${TENSORFLOW_INCLUDE_ROOT})
add_library(
JointEstimator2D SHARED
jointEstimator2D.cpp
cameraControl.cpp
visualization.cpp
${CMAKE_SOURCE_DIR}/src/Tensorflow/tensorflow.cpp
${CMAKE_SOURCE_DIR}/src/Tensorflow/tf_utils.cpp
)
target_link_libraries(JointEstimator2D rt dl m Tensorflow TensorflowFramework )
set_target_properties(JointEstimator2D PROPERTIES DEBUG_POSTFIX "D")
set_target_properties(JointEstimator2D PROPERTIES
ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_SOURCE_DIR}"
LIBRARY_OUTPUT_DIRECTORY "${CMAKE_SOURCE_DIR}"
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_SOURCE_DIR}"
)
find_package(OpenCV REQUIRED)
INCLUDE_DIRECTORIES(${OpenCV_INCLUDE_DIRS})
add_executable(WebcamAnd2DJoints webcamAnd2DJoints.cpp)
target_link_libraries(WebcamAnd2DJoints rt dl m ${OpenCV_LIBRARIES} JointEstimator2D Tensorflow TensorflowFramework)
set_target_properties(WebcamAnd2DJoints PROPERTIES DEBUG_POSTFIX "D")
set_target_properties(WebcamAnd2DJoints PROPERTIES
ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_SOURCE_DIR}"
LIBRARY_OUTPUT_DIRECTORY "${CMAKE_SOURCE_DIR}"
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_SOURCE_DIR}"
)
animation/MocapNET/src/JointEstimator2D/cameraControl.cpp
+433
View File
@@ -0,0 +1,433 @@
#include "cameraControl.hpp"
#include "../MocapNET2/MocapNETLib2/IO/commonSkeleton.hpp"
#include <cstdlib>
const unsigned int noiseThreshold = 30; //pixels
/**
* @brief In order to have the best possible quality we can crop the input frame to only perform detection around the area of the previous skeleton
* This code performs this crop and tries to get the best detection window
* @ingroup demo
* @bug This code is oriented to a single 2D skeleton detected, Multiple skeletons will confuse it and there is no logic to handle them
* @retval 1=Success/0=Failure
*/
int getMaximumCropWindow(
unsigned int * x,
unsigned int * y,
unsigned int * width,
unsigned int * height,
struct boundingBox * bboxExternal,
unsigned int inputWidth2DJointDetector,
unsigned int inputHeight2DJointDetector,
unsigned int fullFrameWidth,
unsigned int fullFrameHeight
)
{
//Make a local copy in case of failure
struct boundingBox bboxLocal = *bboxExternal;
struct boundingBox * bbox = &bboxLocal;
float paddingPercent=0.2; // 0.1 = 10% etc
if ( (bbox!=0) && (bbox->populated) )
{
unsigned int previousX = *x;
unsigned int previousY = *y;
unsigned int previousWidth = *width;
unsigned int previousHeight = *height;
//std::cerr<<"Previous Bounding Box was "<<*x<<","<<*y<<" to "<<*x+*width<<","<<*y+*height<<" \n";
// Bring bounding box to normal coordinates
//------------------------------------------------
bbox->minimumX+= (float) *x;
bbox->maximumX+= (float) *x;
bbox->minimumY+= (float) *y;
bbox->maximumY+= (float) *y;
//Calculate the dimensions of the bounding box body and its center
//----------------------------------------------------------------------------------------------------------
float bodyWidth = bbox->maximumX - bbox->minimumX;
float bodyHeight = bbox->maximumY - bbox->minimumY;
float bodyCenterX = bbox->minimumX + (float) bodyWidth/2;
float bodyCenterY = bbox->minimumY + (float) bodyHeight/2;
//If the body is smaller than our input size lets get some more detail in so we avoid scaling up
//----------------------------------------------------------------------------------------------------------------------------------------------------
if (inputWidth2DJointDetector >bodyWidth )
{
bodyWidth = inputWidth2DJointDetector;
}
if (inputHeight2DJointDetector> bodyHeight )
{
bodyHeight = inputHeight2DJointDetector;
}
//Given that we have our new size and center we can now calculate the new offset from center
//--------------------------------------------------------------------------------------------------------------------------------------------------------
float bodyOffsetX = (float) bodyWidth/2;
float bodyOffsetY = (float) bodyHeight/2;
//And using the new offset derive the new bounding box
//--------------------------------------------------------------------------------------------------------------------------------------------------------
float pixelsPaddedX = paddingPercent * bodyWidth;
float pixelsPaddedY = paddingPercent * bodyHeight;
//We now have a preliminary bounding box BUT the final bounding box we want is a rectangle so we need to keep the biggest dimension
//-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
if ( bodyOffsetX+pixelsPaddedX> bodyOffsetY+pixelsPaddedY)
{
bodyOffsetX = bodyOffsetX + pixelsPaddedX;
bodyOffsetY = bodyOffsetX + pixelsPaddedX;
}
else
{
bodyOffsetX = bodyOffsetY + pixelsPaddedY;
bodyOffsetY = bodyOffsetY + pixelsPaddedY;
}
//We now have a rectangle bounding box
//----------------------------------------------------------------
bbox->minimumX = bodyCenterX - bodyOffsetX;
bbox->maximumX = bodyCenterX + bodyOffsetX;
bbox->minimumY = bodyCenterY - bodyOffsetY;
bbox->maximumY = bodyCenterY + bodyOffsetY;
//We do a last bounds check
//--------------------------------------------------------------------------------------------------------------------------------------------------------
signed int limitLX=0,limitLY=0,limitRX=0,limitRY=0;
if (bbox->minimumX < 0 )
{
limitLX=-1*bbox->minimumX;
bbox->minimumX = 0;
}
if (bbox->minimumY < 0 )
{
limitLY=-1*bbox->minimumY;
bbox->minimumY = 0;
}
if (bbox->maximumX > fullFrameWidth )
{
limitRX=bbox->maximumX-fullFrameWidth;
bbox->maximumX = fullFrameWidth;
}
if (bbox->maximumY > fullFrameHeight )
{
limitRY=bbox->maximumY-fullFrameHeight;
bbox->maximumY = fullFrameHeight;
}
//We now have a preliminary bounding box the rectangle might be in a corner and fall out of bounds
//--------------------------------------------------------------------------------------------------------------------------------------------------------
//We are maybe constrained left/up add what we lost right/down..!
if (limitLX>0)
{
bbox->maximumX += limitLX;
}
if (limitLY>0)
{
bbox->maximumY += limitLY;
}
//We are maybe constrained right/down we subtract what we lost left/up ..!
if (limitRX>0)
{
bbox->minimumX -= limitRX;
}
if (limitRY>0)
{
bbox->minimumY -= limitRY;
}
if ( (limitLX!=0) || (limitLY!=0) || (limitRX!=0) || (limitRY!=0) )
{
//We have failed to initially resolve the bounding box..!
//std::cerr<<"Initial resolve the bounding box X1:"<<limitLX<<",Y1:"<<limitLY<<",X2:"<<limitRX<<",Y2:"<<limitRY<<"..\n";
//return 0;
}
//We one last time check for bound limits
limitLX=0,limitLY=0,limitRX=0,limitRY=0;
if (bbox->minimumX < 0 )
{
limitLX=1;
bbox->minimumX = 0;
}
if (bbox->minimumY < 0 )
{
limitLY=1;
bbox->minimumY = 0;
}
if (bbox->maximumX > fullFrameWidth )
{
limitRX=1;
bbox->maximumX = fullFrameWidth;
}
if (bbox->maximumY > fullFrameHeight )
{
limitRY=1;
bbox->maximumY = fullFrameHeight;
}
if ( (limitLX!=0) || (limitLY!=0) || (limitRX!=0) || (limitRY!=0) )
{
//We have failed to resolve the bounding box..!
//std::cerr<<"We have failed to resolve the bounding box X1:"<<limitLX<<",Y1:"<<limitLY<<",X2:"<<limitRX<<",Y2:"<<limitRY<<"..\n";
//std::cerr<<"bounding box was X1:"<<bbox->minimumX<<",Y1:"<<bbox->minimumY<<",X2:"<<bbox->maximumX<<",Y2:"<<bbox->maximumY<<"..\n";
return 0;
}
//And our output is ready for use..!
//--------------------------------------------------------------------------------------------------------------------------------------------------------
if (
( abs(previousX - (unsigned int) bbox->minimumX ) > noiseThreshold ) ||
( abs(previousY - (unsigned int) bbox->minimumY ) > noiseThreshold ) ||
( abs(previousWidth - (unsigned int) ( bbox->maximumX - bbox->minimumX ) ) > 2*noiseThreshold ) ||
( abs(previousHeight - (unsigned int) ( bbox->maximumY - bbox->minimumY ) ) > 2*noiseThreshold )
)
{
*x=(unsigned int) bbox->minimumX ;
*y=(unsigned int) bbox->minimumY ;
*width = (unsigned int) bbox->maximumX - bbox->minimumX;
*height= (unsigned int) bbox->maximumY - bbox->minimumY;
if (*width == *height +1 )
{
*width = *height;
}
//---------------------------------------------------------------------------------------------------------------------------------------------------------------
//std::cerr<<"New MAX Bounding Box is "<<*x<<","<<*y<<" to "<<*x+*width<<","<<*y+*height<<" Image("<<fullFrameWidth<<","<<fullFrameHeight<<")\n";
if ( (*width!=0) && (*height!=0) )
{
//Success..!
*bboxExternal=bboxLocal;
return 1;
}
}
else
{
fprintf(stderr,"Bounding box is too similar with previous so keeping previous..\n");
//std::cerr<<"New MAX Bounding Box would have been "<< bbox->minimumX<<","<< bbox->minimumY<<" to "<< bbox->maximumX<<","<< bbox->maximumY<<" Image("<<fullFrameWidth<<","<<fullFrameHeight<<")\n";
}
}
return 0;
}
/**
* @brief In order to have the best possible quality we can crop the input frame to only perform detection around the area of the previous skeleton
* This code performs this crop and tries to get the best detection window
* @ingroup demo
* @bug This code is oriented to a single 2D skeleton detected, Multiple skeletons will confuse it and there is no logic to handle them
* @retval 1=Success/0=Failure
*/
int getBestCropWindow(
int maximumCrop,
unsigned int * x,
unsigned int * y,
unsigned int * width,
unsigned int * height,
struct boundingBox * bbox,
unsigned int inputWidth2DJointDetector,
unsigned int inputHeight2DJointDetector,
unsigned int fullFrameWidth,
unsigned int fullFrameHeight
)
{
//fprintf(stderr,"getBestCropWindow(%u,%u,%u,%u)\n",*x,*y,*width,*height);
//fprintf(stderr,"input image size is %u,%u\n",fullFrameWidth,fullFrameHeight);
if (bbox==0) { return 0; }
if (maximumCrop)
{
if ( getMaximumCropWindow(
x,
y,
width,
height,
bbox,
inputWidth2DJointDetector,
inputHeight2DJointDetector,
fullFrameWidth,
fullFrameHeight
)
)
{
return 1;
}
//If we fail to get Maximum Crop Window we will settle with normal crop window
}
//fprintf(stderr,"Previous crop started at %u,%u\n", *x,*y);
if ( (bbox!=0) && (bbox->populated) )
{
//fprintf(stderr,"This means that the bounding box (%0.2f,%0.2f) -> (%0.2f,%0.2f)\n",bbox->minimumX,bbox->minimumY,bbox->maximumX,bbox->maximumY);
bbox->minimumX+=(float) *x;
bbox->maximumX+=(float) *x;
bbox->minimumY+=(float) *y;
bbox->maximumY+=(float) *y;
//fprintf(stderr,"is actually (%0.2f,%0.2f) -> (%0.2f,%0.2f)\n",bbox->minimumX,bbox->minimumY,bbox->maximumX,bbox->maximumY);
}
unsigned int dimension = fullFrameHeight;
float bodyWidth = bbox->maximumX - bbox->minimumX;
float bodyHeight = bbox->maximumY - bbox->minimumY;
//fprintf(stderr,"Body starts at %0.2f and ends at %0.2f for a total of %0.2f pixels\n", bbox->minimumX , bbox->maximumX , bbox->maximumX-bbox->minimumX);
if (fullFrameWidth>=fullFrameHeight)
{
//fprintf(stderr,"Since the whole frame has %u pixels\n",fullFrameWidth);
//fprintf(stderr,"And we will use the dimension of the Y axis aka %u pixels to crop a rectangle\n",fullFrameHeight);
if ( (bbox!=0) && (bbox->populated) )
{
dimension = fullFrameHeight;
//TODO:
//if (bodyHeight<dimension) { dimension = bodyHeight; }
//if (dimension<inputHeight2DJointDetector) { dimension = inputHeight2DJointDetector; }
*width=dimension;
*height=dimension;
//-------------------------------------------------------------------------------------------------------
//-------------------------------------------------------------------------------------------------------
// Center on X axis..
//-------------------------------------------------------------------------------------------------------
//-------------------------------------------------------------------------------------------------------
float bodyCenterX = bbox->minimumX + (float) bodyWidth/2;
//fprintf(stderr,"The center X of the body lies at %0.2f\n",bodyCenterX);
//fprintf(stderr,"We can afford %u pixels left and right of the body center \n",(unsigned int) *width/2);
float cropStartX = bodyCenterX - *width/2;
if (cropStartX<0)
{
cropStartX=0; //Overflow..
}
if (cropStartX>fullFrameWidth-fullFrameHeight)
{
cropStartX=fullFrameWidth-fullFrameHeight;
}
//Neural Networks cause flicker, if we don't have exactly the same
//bounding box it's ok just keep the previous, it will help with flicker a lot..
if ( abs(*x - (unsigned int) cropStartX) > noiseThreshold )
{
*x = (unsigned int) cropStartX;
}
//fprintf(stderr,"This means starting at %u and ending at %u\n",(unsigned int) *x, *x+*width);
/*
//-------------------------------------------------------------------------------------------------------
//-------------------------------------------------------------------------------------------------------
// Center on Y axis..
//-------------------------------------------------------------------------------------------------------
//-------------------------------------------------------------------------------------------------------
float bodyCenterY = bbox->minimumY+ (float) bodyHeight/2;
//fprintf(stderr,"The center X of the body lies at %0.2f\n",bodyCenterX);
//fprintf(stderr,"We can afford %u pixels up and down of the body center \n",(unsigned int) *height/2);
float cropStartY = bodyCenterY - *height/2;
if (cropStartY<0) { cropStartY=0; } //Overflow..
if (cropStartY>fullFrameHeight) { cropStartY=fullFrameHeight; }
//Neural Networks cause flicker, if we don't have exactly the same
//bounding box it's ok just keep the previous, it will help with flicker a lot..
if ( abs(*y - (unsigned int) cropStartY) > 4 )
{
*y = (unsigned int) cropStartY;
}
*/
//std::cerr<<"Normal Bounding Box derived "<<*x<<","<<*y<<" to "<<*x+*width<<","<<*y+*height<<" \n";
}
else
{
//No skeleton? just crop in the center..
*x=(fullFrameWidth-fullFrameHeight)/2;
*y=0;
*width=fullFrameHeight;
*height=fullFrameHeight;
}
}
else if (fullFrameWidth<fullFrameHeight)
{
if ( (bbox!=0) && (bbox->populated) )
{
dimension = fullFrameWidth;
//TODO:
//if (bodyHeight<dimension) { dimension = bodyHeight; }
//if (dimension<inputHeight2DJointDetector) { dimension = inputHeight2DJointDetector; }
*width=dimension;
*height=dimension;
//-------------------------------------------------------------------------------------------------------
//-------------------------------------------------------------------------------------------------------
// Center on X axis..
//-------------------------------------------------------------------------------------------------------
//-------------------------------------------------------------------------------------------------------
float bodyCenterY = bbox->minimumY + (float) bodyHeight/2;
//fprintf(stderr,"The center X of the body lies at %0.2f\n",bodyCenterX);
//fprintf(stderr,"We can afford %u pixels left and right of the body center \n",(unsigned int) *width/2);
float cropStartY = bodyCenterY - *width/2;
if (cropStartY<0)
{
cropStartY=0; //Overflow..
}
if (cropStartY>fullFrameHeight-fullFrameWidth)
{
cropStartY=fullFrameHeight-fullFrameWidth;
}
//Neural Networks cause flicker, if we don't have exactly the same
//bounding box it's ok just keep the previous, it will help with flicker a lot..
if ( abs(*y - (unsigned int) cropStartY) > noiseThreshold )
{
*y = (unsigned int) cropStartY;
}
}
else
{
//No skeleton? just crop in the center..
*x=0;
*y=(fullFrameHeight-fullFrameWidth)/2;
*width=fullFrameWidth;
*height=fullFrameWidth;
}
}
if (
( *x + *width > fullFrameWidth ) ||
( *y + *height > fullFrameHeight )
)
{
//We failed..!
*x=0;
*y=0;
*width=fullFrameWidth;
*height=fullFrameWidth;
return 0;
}
return 1;
}
animation/MocapNET/src/JointEstimator2D/cameraControl.hpp
+19
View File
@@ -0,0 +1,19 @@
#pragma once
#include "opencv2/opencv.hpp"
using namespace cv;
int getBestCropWindow(
int maximumCrop,
unsigned int * x,
unsigned int * y,
unsigned int * width,
unsigned int * height,
struct boundingBox * bbox,
unsigned int inputWidth2DJointDetector,
unsigned int inputHeight2DJointDetector,
unsigned int fullFrameWidth,
unsigned int fullFrameHeight
);
animation/MocapNET/src/JointEstimator2D/jointEstimator2D.cpp
+725
View File
@@ -0,0 +1,725 @@
#include "jointEstimator2D.hpp"
#include <vector>
#include <algorithm> // std::sort
#include <stdio.h>
#include <string.h>
#include <time.h>
unsigned long tickBaseJointEstimator = 0;
unsigned long GetTickCountMicrosecondsJointEstimator()
{
struct timespec ts;
if ( clock_gettime(CLOCK_MONOTONIC,&ts) != 0)
{
return 0;
}
if (tickBaseJointEstimator==0)
{
tickBaseJointEstimator = ts.tv_sec*1000000 + ts.tv_nsec/1000;
return 0;
}
return ( ts.tv_sec*1000000 + ts.tv_nsec/1000 ) - tickBaseJointEstimator;
}
int dumpHeatmapToCSVFile(const char * filename,struct JointEstimator2D * jnet,std::vector<float> heatmap)
{
unsigned int width = jnet->heatmapWidth2DJointDetector;
unsigned int height = jnet->heatmapHeight2DJointDetector;
FILE * fp = fopen(filename,"w");
if (fp!=0)
{
//---------------------------
// Header
//---------------------------
for (int x=0; x<width; x++)
{
if (x!=0)
{
fprintf(fp,",");
}
fprintf(fp,"x=%u",x);
}
fprintf(fp,"\n");
//---------------------------
//---------------------------
// Body
//---------------------------
unsigned int i=0;
for (int y=0; y<height; y++)
{
for (int x=0; x<width; x++)
{
if (x!=0)
{
fprintf(fp,",");
}
if (heatmap[i]>0.01)
{
fprintf(fp,"%f",heatmap[i]);
}
else if (-0.01>heatmap[i])
{
fprintf(fp,"%f",heatmap[i]);
}
else
{
fprintf(fp,"0");
}
++i;
}
fprintf(fp,"\n");
}
//---------------------------
fclose(fp);
return 1;
}
return 0;
}
int loadJointEstimator2D(struct JointEstimator2D * jnet,int joint2DEstimatorSelected,int usePAFs,unsigned int forceCPU)
{
memset(jnet,0,sizeof(struct JointEstimator2D));
const char networkPathOpenPoseMiniStatic[]="dataset/combinedModel/openpose_model.pb";
const char networkPathVnectStatic[]="dataset/combinedModel/vnect_sm_pafs_8.1k.pb";
const char networkPathFORTHStatic[]="dataset/combinedModel/mobnet2_tiny_vnect_sm_1.9k.pb";
jnet->networkPath = (char*) networkPathFORTHStatic;
jnet->joint2DSensitivityPercent=30;
char networkInputLayer[]="input_1";
char networkOutputLayer[]="k2tfout_0";
jnet->numberOfOutputTensors = 3;
jnet->heatmapWidth2DJointDetector= 46;
jnet->heatmapHeight2DJointDetector= 46;
jnet->inputWidth2DJointDetector = 368;
jnet->inputHeight2DJointDetector = 368;
jnet->numberOfHeatmaps = 19;
//-----------------------------------------------------------------------------
switch (joint2DEstimatorSelected)
{
case JOINT_2D_ESTIMATOR_FORTH :
jnet->networkPath=(char*) networkPathFORTHStatic;
networkOutputLayer[8]='0';
jnet->joint2DSensitivityPercent=30;
jnet->numberOfOutputTensors = 3;
break;
case JOINT_2D_ESTIMATOR_VNECT :
jnet->networkPath = (char*) networkPathVnectStatic;
networkOutputLayer[8]='1';
jnet->joint2DSensitivityPercent=20;
jnet->numberOfOutputTensors = 4;
break;
case JOINT_2D_ESTIMATOR_OPENPOSE :
jnet->networkPath=(char*) networkPathOpenPoseMiniStatic;
networkOutputLayer[8]='1';
jnet->joint2DSensitivityPercent=40;
jnet->numberOfOutputTensors = 4;
break;
default :
fprintf(stderr,"No 2D Joint Estimator Selected..\n");
break;
};
//-----------------------------------------------------------------------------
if (
loadTensorflowInstance(
&jnet->network,
jnet->networkPath,
networkInputLayer,
networkOutputLayer,
forceCPU
)
)
{
return 1;
}
//-----------------------------------------------------------------------------
return 0;
}
int unloadJointEstimator2D(struct JointEstimator2D * jnet)
{
return unloadTensorflow(&jnet->network);
}
int sortBlobsBasedOnMaximumActivation(struct HeatmapBlob blobA,struct HeatmapBlob blobB)
{
return blobA.peakValue > blobB.peakValue;
}
int updateSubpixelPeak(struct JointEstimator2D * jnet,struct HeatmapBlob * blob,std::vector<float> heatmap,unsigned int width,unsigned int height)
{
//Uncomment to disable subpixel code..
//blob->subpixelPeakX = (float) blob->peakX;
//blob->subpixelPeakY = (float) blob->peakY;
//return 0;
#define DEBUG_SUBPIXEL_GRADIENT 0
#if DEBUG_SUBPIXEL_GRADIENT
std::vector<float> gradientX(width*height);
std::vector<float> gradientY(width*height);
#endif
if (blob->peakValue==0.0)
{
return 0;
}
float vX=(float) blob->peakX;
float vY=(float) blob->peakY;
float normalizedValue=0.0;
unsigned int location=0;
for (signed int y=0; y<blob->height; y++)
{
for (signed int x=0; x<blob->width; x++)
{
//fprintf(stderr,"Blob element (x=%u,y=%u) ",x,y);
location = (width * (y+blob->y)) + (x+blob->x);
//fprintf(stderr,"Location %u",location);
if (location<heatmap.size())
{
normalizedValue=(float) heatmap[location] / blob->peakValue;
normalizedValue= normalizedValue / ( blob->width * blob->height);
//fprintf(stderr,"NormalValue %0.2f",normalizedValue);
signed int peakInRelationToBlobX = (signed int) ((signed int) blob->peakX - (signed int) blob->x);
signed int peakInRelationToBlobY = (signed int) ((signed int) blob->peakY - (signed int) blob->y);
signed int relativePositionToPeakX = (signed int) ( (signed int) x - (signed int) peakInRelationToBlobX);
signed int relativePositionToPeakY = (signed int) ( (signed int) y - (signed int) peakInRelationToBlobY);
//fprintf(stderr,"Ppeak %d,%d",posX,posY);
float nX=(float) relativePositionToPeakX*normalizedValue;
float nY=(float) relativePositionToPeakY*normalizedValue;
//fprintf(stderr,"Add %f,%f\n",nX,nY);
#if DEBUG_SUBPIXEL_GRADIENT
gradientX[location]=nX;
gradientY[location]=nY;
#endif
if ( (nX!=nX) || (nY!=nY) )
{
//Handle NaN
}
else
{
//fprintf(stderr,"%f,%f ",nX,nY);
vX=vX+nX;
vY=vY+nY;
}
}
else
{
fprintf(stderr,"updateSubpixelPeak overflow..\n");
return 0;
}
}
}
#if DEBUG_SUBPIXEL_GRADIENT
char filename[512];
snprintf(filename,512,"GradientX.csv");
dumpHeatmapToCSVFile(filename,jnet,gradientX);
snprintf(filename,512,"GradientY.csv");
dumpHeatmapToCSVFile(filename,jnet,gradientY);
snprintf(filename,512,"Values.csv");
dumpHeatmapToCSVFile(filename,jnet,heatmap);
#endif
blob->subpixelPeakX = vX;
blob->subpixelPeakY = vY;
return 1;
}
int isNeighborWithBlob(char * blobLabels,int x,int y,int width,int height)
{
for (int iY=-1; iY<1; iY++)
{
for (int iX=-1; iX<=1; iX++)
{
if ( (iX==0) && (iY==0) )
{
}
else if (
(iX+x>=0) &&
(iY+y>=0) &&
(iX+x<width) &&
(iY+y<height)
)
{
int location = width * (y+iY) + (x+iX);
if (blobLabels[location]!=0)
{
return blobLabels[location];
}
}
}
}
return 0;
}
std::vector<struct HeatmapBlob> getBlobsFromHeatmap(struct JointEstimator2D * jnet,unsigned int heatmapID,std::vector<float> heatmap)
{
unsigned int width = jnet->heatmapWidth2DJointDetector;
unsigned int height = jnet->heatmapHeight2DJointDetector;
struct HeatmapBlob emptyBlob;
emptyBlob.x=width+1;
emptyBlob.y=height+1;
emptyBlob.width=0;
emptyBlob.height=0;
emptyBlob.width=0;
emptyBlob.peakX=0;
emptyBlob.peakY=0;
emptyBlob.peakValue=0.0;
emptyBlob.subpixelPeakX=0.0;
emptyBlob.subpixelPeakY=0.0;
std::vector<struct HeatmapBlob> blobsEncountered;
char blobLabels[width*height]= {0};
unsigned int currentBlobLabel=1;
int onHorizontalBlob=0;
int labelID=0;
unsigned int i=0;
float threshold = 0.1; //(float) jnet->joint2DSensitivityPercent/100;
for (int y=0; y<height; y++)
{
for (int x=0; x<width; x++)
{
if ( heatmap[i] > threshold )
{
if (!onHorizontalBlob)
{
//We were not on a horizontal blob so we need to search..!
labelID=isNeighborWithBlob(blobLabels,x,y,width,height);
if (!labelID)
{
//Our search did not give us a result
//This is a new blob disconnected from previous blobs..!
blobLabels[i]=currentBlobLabel;
onHorizontalBlob=currentBlobLabel;
labelID=currentBlobLabel-1;
++currentBlobLabel;
blobsEncountered.push_back(emptyBlob);
onHorizontalBlob=1;
}
else
{
//Already Existing Blob found we will add this pixel to it
onHorizontalBlob=labelID;
blobLabels[i]=onHorizontalBlob;
labelID=labelID-1;
}
}
else
{
//If we are already on a horizontal blob dont do expensive search
blobLabels[i]=onHorizontalBlob;
labelID=onHorizontalBlob-1;
}
if (labelID<blobsEncountered.size())
{
//fprintf(stderr,"Hit at %u,%u\n",x,y);
//fprintf(stderr,"%f ",heatmap[i]);
//Update Blob Start at X Axis
if (x<blobsEncountered[labelID].x)
{
blobsEncountered[labelID].x=x;
}
//Update Blob Start at Y Axis
if (y<blobsEncountered[labelID].y)
{
blobsEncountered[labelID].y=y;
}
//Update Blob Width
int thisLineWidth=1+x-blobsEncountered[labelID].x;
if (thisLineWidth>blobsEncountered[labelID].width)
{
blobsEncountered[labelID].width=thisLineWidth;
}
//Update Blob Height
int thisLineHeight=1+y-blobsEncountered[labelID].y;
if (thisLineHeight>blobsEncountered[labelID].height)
{
blobsEncountered[labelID].height=thisLineHeight;
}
//Update Peak to use as center later..
if (blobsEncountered[labelID].peakValue<heatmap[i])
{
blobsEncountered[labelID].peakValue=heatmap[i];
blobsEncountered[labelID].peakX=x;
blobsEncountered[labelID].peakY=y;
}
}
}//Above activation threshold..
else
{
//We just left a blob..
onHorizontalBlob=0;
}
++i;
} //End of X loop
//We just left a line so the blob has ended..
onHorizontalBlob=0;
} //End of Y loop
for (int blobID=0; blobID<blobsEncountered.size(); blobID++)
{
updateSubpixelPeak(jnet,&blobsEncountered[blobID],heatmap,width,height);
}
//Make sure the best value is first..
std::sort(blobsEncountered.begin(), blobsEncountered.end(),sortBlobsBasedOnMaximumActivation);
/*
fprintf(stderr,"Heatmap %u %s--------\n",heatmapID,UT_COCOBodyNames[heatmapID]);
for (int blobID=0; blobID<blobsEncountered.size(); blobID++)
{
fprintf(stderr,"Blob %u/%lu - ",blobID,blobsEncountered.size());
fprintf(stderr,"x=%u,y=%u(%ux%u) "
,blobsEncountered[blobID].x,blobsEncountered[blobID].y
,blobsEncountered[blobID].width,blobsEncountered[blobID].height
);
fprintf(stderr,"pX=%u,pY=%u>",blobsEncountered[blobID].peakX,blobsEncountered[blobID].peakY);
fprintf(stderr,"%0.2f",blobsEncountered[blobID].peakValue);
fprintf(stderr," subpixel(%0.2f,%0.2f)",blobsEncountered[blobID].subpixelPeakX,blobsEncountered[blobID].subpixelPeakY);
fprintf(stderr,"\n");
}
*/
return blobsEncountered;
}
int estimate2DSkeletonsFromHeatmaps(struct JointEstimator2D * jnet,struct Skeletons2DDetected * result,std::vector<std::vector<float> > heatmaps)
{
//char filename[512];
//fprintf(stderr,"New Frame:\n");
unsigned long startTime = GetTickCountMicrosecondsJointEstimator();
int blobsProceesed=0;
for (int heatmapID=0; heatmapID<heatmaps.size(); heatmapID++)
{
unsigned int body25ID = heatmapCorrespondenceToBODY25[heatmapID];
if (heatmapID!=UT_COCO_Bkg)
{
result->skeletons[0].body.joint2D[body25ID].x = 0;
result->skeletons[0].body.joint2D[body25ID].y = 0;
result->skeletons[0].body.active[body25ID]=0.0;
//--------------------------------------------------------
std::vector<struct HeatmapBlob> blobs = getBlobsFromHeatmap(jnet,heatmapID,heatmaps[heatmapID]);
if (blobs.size()>0)
{
result->skeletons[0].body.isPopulated=1;
float x=(float) blobs[0].subpixelPeakX/ jnet->heatmapWidth2DJointDetector;
float y=(float) blobs[0].subpixelPeakY/ jnet->heatmapHeight2DJointDetector;
result->skeletons[0].body.joint2D[body25ID].x = x;
result->skeletons[0].body.joint2D[body25ID].y = y;
if ( (x!=0) && (y!=0) ) {
result->skeletons[0].body.active[body25ID]=1.0;
}
if (blobsProceesed==0)
{
//Update Bbox Min
result->skeletons[0].body.bbox2D[0].x = x;
result->skeletons[0].body.bbox2D[0].y = y;
//Update Bbox Max
result->skeletons[0].body.bbox2D[1].x = x;
result->skeletons[0].body.bbox2D[1].y = y;
}
else
{
// if Min X > current X
if (result->skeletons[0].body.bbox2D[0].x > x )
{
result->skeletons[0].body.bbox2D[0].x =x;
}
// if Min Y > current Y
if (result->skeletons[0].body.bbox2D[0].y > y )
{
result->skeletons[0].body.bbox2D[0].y =y;
}
// if Max X < current X
if (result->skeletons[0].body.bbox2D[1].x < x )
{
result->skeletons[0].body.bbox2D[1].x =x;
}
// if Max Y < current Y
if (result->skeletons[0].body.bbox2D[1].y < y )
{
result->skeletons[0].body.bbox2D[1].y =y;
}
}
++blobsProceesed;
}
//--------------------------------------------------------
//snprintf(filename,512,"heatmap_%u.csv",heatmapID);
//dumpHeatmapToCSVFile(filename,jnet,heatmaps[0]);
//--------------------------------------------------------
}
}
/*
The following joints are not covered by the current networks
BODY25_MidHip,
BODY25_LBigToe,
BODY25_LSmallToe,
BODY25_LHeel,
BODY25_RBigToe,
BODY25_RSmallToe,
BODY25_RHeel,
*/
//Mid hip can be approximated by LHip and RHip
//--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
if (
(result->skeletons[0].body.joint2D[BODY25_RHip].x!=0) && (result->skeletons[0].body.joint2D[BODY25_LHip].x!=0) &&
(result->skeletons[0].body.joint2D[BODY25_RHip].y!=0) && (result->skeletons[0].body.joint2D[BODY25_LHip].y!=0)
)
{
result->skeletons[0].body.joint2D[BODY25_MidHip].x = (float) (result->skeletons[0].body.joint2D[BODY25_RHip].x+result->skeletons[0].body.joint2D[BODY25_LHip].x)/2;
result->skeletons[0].body.joint2D[BODY25_MidHip].y = (float) (result->skeletons[0].body.joint2D[BODY25_RHip].y+result->skeletons[0].body.joint2D[BODY25_LHip].y)/2;
result->skeletons[0].body.jointAccuracy[BODY25_MidHip] = (result->skeletons[0].body.jointAccuracy[BODY25_RHip]+result->skeletons[0].body.jointAccuracy[BODY25_LHip])/2;
result->skeletons[0].body.active[BODY25_MidHip] = 1.0;
//fprintf(stderr,"Mid hip populated and resides @ (%0.2f,%0.2f)\n",result->skeletons[0].body.joint2D[BODY25_MidHip].x,result->skeletons[0].body.joint2D[BODY25_MidHip].y);
} else
{
result->skeletons[0].body.joint2D[BODY25_MidHip].x = 0;
result->skeletons[0].body.joint2D[BODY25_MidHip].y = 0;
result->skeletons[0].body.jointAccuracy[BODY25_MidHip] = 0;
result->skeletons[0].body.active[BODY25_MidHip] = 0.0;
//fprintf(stderr,"Cannot populate Mid hip, this will degrade output!\n");
}
//--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
result->numberOfSkeletonsDetected=1;
unsigned long endTime = GetTickCountMicrosecondsJointEstimator();
// fprintf(stderr,"Resolving joint subpixel peaks took %lu microseconds\n",endTime-startTime);
return 1;
}
int convertNormalized2DJointsToOriginalImageCoordinates(
struct JointEstimator2D * jnet,
float * x,
float * y,
int correctOffset
)
{
//It is very common for points to be empty, they cannot be normalized so its best if we have
//a quick path for them..
if ( (*x==0.0) && (*y==0.0) ) {
return 1;
}
if (*x>1.0) {
*x=1.0;
}
if (*y>1.0) {
*y=1.0;
}
float normalizedX = *x;
float normalizedY = *y;
//fprintf(stderr,"Initial point is %0.2f,%0.2f\n",normalizedX,normalizedY);
float frameOfReferenceOfTensorflowX = normalizedX * jnet->inputWidth2DJointDetector;
float frameOfReferenceOfTensorflowY = normalizedY * jnet->inputHeight2DJointDetector;
//fprintf(stderr,"Tensorflow point is %0.2f,%0.2f\n",frameOfReferenceOfTensorflowX,frameOfReferenceOfTensorflowY);
float scaleFromTensorflowToBoundingBoxX = (float) jnet->crop.croppedDimensionWidth / jnet->inputWidth2DJointDetector;
float scaleFromTensorflowToBoundingBoxY = (float) jnet->crop.croppedDimensionHeight / jnet->inputHeight2DJointDetector;
*x = /*jnet->crop.offsetX +*/ (frameOfReferenceOfTensorflowX * scaleFromTensorflowToBoundingBoxX);
*y = /*jnet->crop.offsetY +*/ (frameOfReferenceOfTensorflowY * scaleFromTensorflowToBoundingBoxY);
if (correctOffset)
{
*x += jnet->crop.offsetX;
*y += jnet->crop.offsetY;
}
//fprintf(stderr,"Final point is %0.2f,%0.2f\n",*x,*y);
return 1;
}
std::vector<std::vector<float> > getHeatmaps(struct JointEstimator2D * jnet,unsigned char * rgbData,unsigned int width,unsigned int height)
{
// pass the frame to the Estimator
unsigned long startTime = GetTickCountMicrosecondsJointEstimator();
std::vector<std::vector<float> > joint2DOutput = predictTensorflowOnArrayOfHeatmaps(
&jnet->network,
(unsigned int) width,
(unsigned int) height,
(float*) rgbData,
jnet->heatmapWidth2DJointDetector,
jnet->heatmapHeight2DJointDetector,
jnet->numberOfOutputTensors
);
unsigned long endTime = GetTickCountMicrosecondsJointEstimator();
//fprintf(stderr,"Running 2D joint estimator took %lu microseconds\n",endTime-startTime);
return joint2DOutput;
}
int estimate2DSkeletonsFromImage(struct JointEstimator2D * jnet,struct Skeletons2DDetected * result,unsigned char * rgbData,unsigned int width,unsigned int height)
{
/*
unsigned long startTime = GetTickCountMicroseconds();
std::vector<cv::Point_<float> > pointsOf2DSkeleton = predictAndReturnSingleSkeletonOf2DCOCOJoints(
&jnet->network,
bgr,
minThreshold,
visualize,
saveVisualization,
frameNumber,
inputWidth2DJointDetector,
inputHeight2DJointDetector,
heatmapWidth2DJointDetector,
heatmapHeight2DJointDetector,
numberOfHeatmaps,
numberOfOutputTensors
);
unsigned long endTime = GetTickCountMicroseconds();
*fps = convertStartEndTimeFromMicrosecondsToFPS(startTime,endTime);
*/
// pass the frame to the Estimator
std::vector<std::vector<float> > joint2DOutput = predictTensorflowOnArrayOfHeatmaps(
&jnet->network,
(unsigned int) width,
(unsigned int) height,
(float*) rgbData,
jnet->heatmapWidth2DJointDetector,
jnet->heatmapHeight2DJointDetector,
jnet->numberOfOutputTensors
);
return 0;
}
int restore2DJointsToInputFrameCoordinates(struct JointEstimator2D * jnet,struct Skeletons2DDetected * input)
{
for (unsigned int skID=0; skID<input->numberOfSkeletonsDetected; skID++)
{
for (int i=0; i<BODY25_PARTS; i++)
{
//fprintf(stderr,"normalized(%0.2f,%0.2f) ",input->skeletons[skID].body.joint2D[i].x,input->skeletons[skID].body.joint2D[i].y);
convertNormalized2DJointsToOriginalImageCoordinates(jnet,&input->skeletons[skID].body.joint2D[i].x,&input->skeletons[skID].body.joint2D[i].y,1);
//fprintf(stderr,"regular(%0.2f,%0.2f) ",input->skeletons[skID].body.joint2D[i].x,input->skeletons[skID].body.joint2D[i].y);
}
/* //Currently dont have hand and head networks..!
for (int i=0; i<COCO_HAND_PARTS; i++)
{
convertNormalized2DJointsToOriginalImageCoordinates(jnet,&input->skeletons[skID].leftHand.joint2D[i].x,&input->skeletons[skID].leftHand.joint2D[i].y,1);
}
for (int i=0; i<COCO_HAND_PARTS; i++)
{
convertNormalized2DJointsToOriginalImageCoordinates(jnet,&input->skeletons[skID].rightHand.joint2D[i].x,&input->skeletons[skID].rightHand.joint2D[i].y,1);
}
for (int i=0; i<OP_HEAD_PARTS; i++)
{
convertNormalized2DJointsToOriginalImageCoordinates(jnet,&input->skeletons[skID].head.joint2D[i].x,&input->skeletons[skID].head.joint2D[i].y,1);
}
*/
}
return 1;
}
float percentOf2DPointsMissing(struct Skeletons2DDetected * input)
{
if (input->numberOfSkeletonsDetected==0) {
return 100.0;
}
unsigned int jointsThatExist=0;
unsigned int jointsThatShouldExist=0;
if (input->skeletons[0].body.isPopulated)
{
jointsThatShouldExist=BODY25_PARTS-2;
for (int jID=0; jID<BODY25_PARTS; jID++)
{
if ( input->skeletons[0].body.active[jID] )
{
++jointsThatExist;
}
}
//fprintf(stderr,"%u/%u joints observed\n",jointsThatExist,jointsThatShouldExist);
float percentOf2DPointsThatExist = (float) 100 * jointsThatExist / jointsThatShouldExist;
return (float) 100.0 - percentOf2DPointsThatExist;
}
//fprintf(stderr,"Body is not populated\n");
return 100.0;
}
animation/MocapNET/src/JointEstimator2D/jointEstimator2D.hpp
+296
View File
@@ -0,0 +1,296 @@
#pragma once
/** @file mocapnet.hpp
* @brief The MocapNET C library
* As seen in https://www.youtube.com/watch?v=fH5e-KMBvM0 , the MocapNET network requires two types of input.
* The first is an uncompressed list of (x,y,v) joints and the second an NSDM array. To add to those the output consists of BVH
* frames that must be accompanied by a header. This library internally handles all of these details.
* @author Ammar Qammaz (AmmarkoV)
*/
#include "../Tensorflow/tensorflow.hpp"
#include "../MocapNET2/MocapNETLib2/IO/commonSkeleton.hpp"
#include <iostream>
#include <vector>
/**
* @brief 2D Skeletons Retreieved
*/
struct Skeletons2DDetected
{
unsigned int numberOfSkeletonsDetected;
struct skeletonStructure skeletons[16];
};
struct InputImageCrop
{
unsigned int offsetX;
unsigned int offsetY;
unsigned int croppedDimensionWidth;
unsigned int croppedDimensionHeight;
//cropBBox,
unsigned int frameWidth;
unsigned int frameHeight;
};
/**
* @brief MocapNET consists of separate classes/ensembles that are invoked for particular orientations.
* This structure holds the required tensorflow instances to make MocapNET work.
*/
struct JointEstimator2D
{
struct TensorflowInstance network;
char * networkPath;
float joint2DSensitivityPercent;
unsigned int numberOfOutputTensors;
unsigned int heatmapWidth2DJointDetector;
unsigned int heatmapHeight2DJointDetector;
unsigned int inputWidth2DJointDetector;
unsigned int inputHeight2DJointDetector;
unsigned int numberOfHeatmaps;
struct InputImageCrop crop;
int doCrop;
};
/**
* @brief Each Blob is a rectangular area..
* with one peak..
*/
struct HeatmapBlob
{
unsigned int x;
unsigned int y;
unsigned int width;
unsigned int height;
unsigned int peakX;
unsigned int peakY;
float peakValue;
float subpixelPeakX;
float subpixelPeakY;
};
/**
* @brief This is a list of included Joint 2D Estimators included.
*/
enum JOINT_2D_ESTIMATOR_SELECTED
{
JOINT_2D_ESTIMATOR_NONE=0,
JOINT_2D_ESTIMATOR_FORTH,
JOINT_2D_ESTIMATOR_VNECT,
JOINT_2D_ESTIMATOR_OPENPOSE,
//-----------------------------
JOINT_2D_ESTIMATOR_NUMBER
};
/**
* @brief This is an array of names for the input Joints expected from the utilities.
*/
static const char * UT_COCOBodyNames[] =
{
"Nose", //0
"Neck", //1
"RShoulder", //2
"RElbow", //3
"RWrist", //4
"LShoulder", //5
"LElbow", //6
"LWrist", //7
"RHip", //8
"RKnee", //9
"RAnkle", //10
"LHip", //11
"LKnee", //12
"LAnkle", //13
"REye", //14
"LEye", //15
"REar", //16
"LEar", //17
"Bkg", //18
//=================
"End of Joint Names"
};
/**
* @brief This is a list of included Joint 2D Estimators included.
*/
enum JOINT_2D_ESTIMATOR_PART_LABEL
{
JOINT_2D_ESTIMATOR_PART_OF_NOTHING=0,
JOINT_2D_ESTIMATOR_PART_OF_TORSO,
JOINT_2D_ESTIMATOR_PART_OF_HEAD,
JOINT_2D_ESTIMATOR_PART_OF_LEFT,
JOINT_2D_ESTIMATOR_PART_OF_RIGHT,
//-----------------------------
JOINT_2D_ESTIMATOR_NUMBER_OF_PARTS
};
/**
* @brief We want to distinguish limbs with labels ( to color them )
*/
static const int UT_COCOSkeletonJointsPartLabel[] =
{
// Part of Joint
JOINT_2D_ESTIMATOR_PART_OF_TORSO, //UT_COCO_Nose,
JOINT_2D_ESTIMATOR_PART_OF_TORSO, //UT_COCO_Neck,
JOINT_2D_ESTIMATOR_PART_OF_RIGHT, //UT_COCO_RShoulder,
JOINT_2D_ESTIMATOR_PART_OF_RIGHT, //UT_COCO_RElbow,
JOINT_2D_ESTIMATOR_PART_OF_RIGHT, //UT_COCO_RWrist,
JOINT_2D_ESTIMATOR_PART_OF_LEFT, //UT_COCO_LShoulder,
JOINT_2D_ESTIMATOR_PART_OF_LEFT, //UT_COCO_LElbow,
JOINT_2D_ESTIMATOR_PART_OF_LEFT, //UT_COCO_LWrist,
JOINT_2D_ESTIMATOR_PART_OF_RIGHT, //UT_COCO_RHip,
JOINT_2D_ESTIMATOR_PART_OF_RIGHT, //UT_COCO_RKnee,
JOINT_2D_ESTIMATOR_PART_OF_RIGHT, //UT_COCO_RAnkle,
JOINT_2D_ESTIMATOR_PART_OF_LEFT, //UT_COCO_LHip,
JOINT_2D_ESTIMATOR_PART_OF_LEFT, //UT_COCO_LKnee,
JOINT_2D_ESTIMATOR_PART_OF_LEFT, //UT_COCO_LAnkle,
JOINT_2D_ESTIMATOR_PART_OF_HEAD, //UT_COCO_REye,
JOINT_2D_ESTIMATOR_PART_OF_HEAD, //UT_COCO_LEye,
JOINT_2D_ESTIMATOR_PART_OF_HEAD, //UT_COCO_REar,
JOINT_2D_ESTIMATOR_PART_OF_HEAD, //UT_COCO_LEar,
JOINT_2D_ESTIMATOR_PART_OF_NOTHING //UT_COCO_Bkg
};
/**
* @brief This is a programmer friendly enumerator of joint names expected from the utilities ( hence the prefix UT_ ) .
* Please notice that these are not necessarily the same when converting to different data formats ( COCO / BODY25 / MocapNET etc )
*/
enum UT_COCOSkeletonJoints
{
UT_COCO_Nose=0,
UT_COCO_Neck,
UT_COCO_RShoulder,
UT_COCO_RElbow,
UT_COCO_RWrist,
UT_COCO_LShoulder,
UT_COCO_LElbow,
UT_COCO_LWrist,
UT_COCO_RHip,
UT_COCO_RKnee,
UT_COCO_RAnkle,
UT_COCO_LHip,
UT_COCO_LKnee,
UT_COCO_LAnkle,
UT_COCO_REye,
UT_COCO_LEye,
UT_COCO_REar,
UT_COCO_LEar,
UT_COCO_Bkg,
//---------------------
UT_COCO_PARTS
};
/**
* @brief This is the parent relation map of COCO skeleton joints , we can easily find a joints parent using this enumerator
*/
static const int UT_COCOSkeletonJointsParentRelationMap[] =
{
// Parent Joint
UT_COCO_Nose, //UT_COCO_Nose,
UT_COCO_Nose, //UT_COCO_Neck,
UT_COCO_Neck, //UT_COCO_RShoulder,
UT_COCO_RShoulder, //UT_COCO_RElbow,
UT_COCO_RElbow, //UT_COCO_RWrist,
UT_COCO_Neck, //UT_COCO_LShoulder,
UT_COCO_LShoulder, //UT_COCO_LElbow,
UT_COCO_LElbow, //UT_COCO_LWrist,
UT_COCO_Neck, //UT_COCO_RHip,
UT_COCO_RHip, //UT_COCO_RKnee,
UT_COCO_RKnee, //UT_COCO_RAnkle,
UT_COCO_Neck, //UT_COCO_LHip,
UT_COCO_LHip, //UT_COCO_LKnee,
UT_COCO_LKnee, //UT_COCO_LAnkle,
UT_COCO_Nose, //UT_COCO_REye,
UT_COCO_Nose, //UT_COCO_LEye,
UT_COCO_REye, //UT_COCO_REar,
UT_COCO_LEye, //UT_COCO_LEar,
UT_COCO_Bkg //UT_COCO_Bkg
};
/**
* @brief An array of indexes to the parents of BODY25 skeleton joints
*/
static const int heatmapCorrespondenceToBODY25[] =
{
BODY25_Nose, // UT_COCO_Nose=0,
BODY25_Neck, // UT_COCO_Neck,
BODY25_RShoulder, //UT_COCO_RShoulder,
BODY25_RElbow,// UT_COCO_RElbow,
BODY25_RWrist,// UT_COCO_RWrist,
BODY25_LShoulder,//UT_COCO_LShoulder,
BODY25_LElbow,//UT_COCO_LElbow,
BODY25_LWrist,//UT_COCO_LWrist,
BODY25_RHip,//UT_COCO_RHip,
BODY25_RKnee,//UT_COCO_RKnee,
BODY25_RAnkle,//UT_COCO_RAnkle,
BODY25_LHip,//UT_COCO_LHip,
BODY25_LKnee,//UT_COCO_LKnee,
BODY25_LAnkle,//UT_COCO_LAnkle,
BODY25_REye,//UT_COCO_REye,
BODY25_LEye,//UT_COCO_LEye,
BODY25_REar,//UT_COCO_REar,
BODY25_LEar,//UT_COCO_LEar,
BODY25_Bkg,//UT_COCO_Bkg,
};
unsigned long GetTickCountMicrosecondsJointEstimator();
/**
* @brief Load a 2D Joint Estimator from a .pb file on disk
* @ingroup jointestimator
* @param Pointer to a struct JointEstimator2D that will hold the tensorflow instances on load.
* @param Quality setting, can currently be 1.0 ( highest quality ), 1.5 or 2.0 ( highest performance )
* @param Flag that controls the use of PAFs
* @param Force the usage of CPU for MocapNET ( should be 1 as MocapNET is designed for CPU while GPU handles 2D )
* @retval 1 = Success loading the files , 0 = Failure
*/
int loadJointEstimator2D(struct JointEstimator2D * jnet,int joint2DEstimatorSelected,int usePAFs,unsigned int forceCPU);
int unloadJointEstimator2D(struct JointEstimator2D * jnet);
int convertNormalized2DJointsToOriginalImageCoordinates(
struct JointEstimator2D * jnet,
float * x,
float * y,
int correctOffset
);
int estimate2DSkeletonsFromImage(struct JointEstimator2D * jnet,struct Skeletons2DDetected * result,unsigned char * rgbData,unsigned int width,unsigned int height);
int restore2DJointsToInputFrameCoordinates(struct JointEstimator2D * jnet,struct Skeletons2DDetected * input);
float percentOf2DPointsMissing(struct Skeletons2DDetected * input);
int estimate2DSkeletonsFromHeatmaps(struct JointEstimator2D * jnet,struct Skeletons2DDetected * result,std::vector<std::vector<float> > heatmaps);
std::vector<std::vector<float> > getHeatmaps(struct JointEstimator2D * jnet,unsigned char * rgbData,unsigned int width,unsigned int height);
animation/MocapNET/src/JointEstimator2D/visualization.cpp
+217
View File
@@ -0,0 +1,217 @@
#include "visualization.hpp"
#include "cameraControl.hpp"
int visualizeHeatmaps(struct JointEstimator2D * jointEstimator,std::vector<std::vector<float> > heatmapNNOutput,unsigned int frameNumber)
{
unsigned int cols = jointEstimator->heatmapWidth2DJointDetector;
unsigned int rows = jointEstimator->heatmapHeight2DJointDetector;
unsigned int hm = jointEstimator->numberOfHeatmaps;
std::vector<cv::Mat> heatmaps;
for(int i=0; i<hm; ++i)
{
cv::Mat h(rows,cols, CV_32FC1);
for(int r=0; r<rows; ++r)
{
for(int c=0; c<cols; ++c)
{
int pos = r*cols+c;
h.at<float>(r,c) = heatmapNNOutput[i][pos];
}
}
heatmaps.push_back(h);
}
#define DISPLAY_ALL_HEATMAPS 1
//This code segment will display every returned heatmap in it's own window..
#if DISPLAY_ALL_HEATMAPS
int visualize=1;
if (visualize)
{
unsigned int x=0;
unsigned int y=0;
char windowLabel[128];
snprintf(windowLabel,128,"T");
for(int i=0; i<18; ++i)
{
snprintf(windowLabel,128,"Heatmap %u",i);
if (frameNumber==0)
{
cv::namedWindow(windowLabel,1);
cv::moveWindow(windowLabel, x,y);
++frameNumber;
}
cv::imshow(windowLabel,heatmaps[i]);
y=y+rows+30;
if (y>700)
{
x=x+cols;
y=0;
}
}
}
#endif
return 1;
}
//Basic Skeleton Visualization with big fonts to be legible on cluttered skeletons
void dj_drawExtractedSkeletons(
cv::Mat img,
struct Skeletons2DDetected * sk,
float factorX,
float factorY
)
{
cv::Scalar red = cv::Scalar(0,0,255);
cv::Scalar green = cv::Scalar(0,255,0);
cv::Scalar blue = cv::Scalar(255,0,0);
cv::Scalar color=blue;
//Just the lines ( background layer)
for (int skID=0; skID<sk->numberOfSkeletonsDetected; skID++)
{
for (int i=0; i<UT_COCO_PARTS; i++)
{
unsigned int jointID = heatmapCorrespondenceToBODY25[i%18];
struct point2D * jointPoint = & sk->skeletons[skID].body.joint2D[jointID];
unsigned int parentID = heatmapCorrespondenceToBODY25[UT_COCOSkeletonJointsParentRelationMap[i%18]];
if (parentID!=jointID)
{
struct point2D * parentPoint = & sk->skeletons[skID].body.joint2D[parentID];
if (
(jointPoint->x > 0) &&
(jointPoint->y > 0) &&
(parentPoint->x > 0) &&
(parentPoint->y > 0)
)
{
cv::Point_<float> jointPointCV;
jointPointCV.x = jointPoint->x * factorX;
jointPointCV.y = jointPoint->y * factorY;
cv::Point_<float> parentPointCV;
parentPointCV.x = parentPoint->x * factorX;
parentPointCV.y = parentPoint->y * factorY;
switch (UT_COCOSkeletonJointsPartLabel[i%18])
{
case JOINT_2D_ESTIMATOR_PART_OF_TORSO:
case JOINT_2D_ESTIMATOR_PART_OF_HEAD:
color=blue;
break;
case JOINT_2D_ESTIMATOR_PART_OF_LEFT:
color=red;
break;
case JOINT_2D_ESTIMATOR_PART_OF_RIGHT:
color=green;
break;
};
cv::line(img,jointPointCV,parentPointCV,color,2.0);
}
}
}
}
//Just the points and text ( foreground )
char textInfo[512];
for (int skID=0; skID<sk->numberOfSkeletonsDetected; skID++)
{
for (int i=0; i<UT_COCO_PARTS; i++)
{
unsigned int jointID = heatmapCorrespondenceToBODY25[i%18];
struct point2D * jointPoint = & sk->skeletons[skID].body.joint2D[jointID];
cv::Point_<float> jointPointCV;
jointPointCV.x = jointPoint->x * factorX;
jointPointCV.y = jointPoint->y * factorY;
if ( (jointPointCV.x>0) && (jointPointCV.y>0) )
{
cv::circle(img,jointPointCV,3,cv::Scalar(255,0,0),3,8,0);
snprintf(textInfo,512,"%s(%u)",UT_COCOBodyNames[i],i);
cv::putText(img, textInfo , jointPointCV, cv::FONT_HERSHEY_DUPLEX, 0.3, cv::Scalar::all(255), 1,8);
}
}
}
}
int cropAndResizeCVMatToMatchSkeleton(
struct JointEstimator2D * jest,
cv::Mat & frame,
struct Skeletons2DDetected * sk
)
{
int tryForMaximumCrop=0;
int result=0;
jest->crop.frameWidth=frame.size().width;
jest->crop.frameHeight=frame.size().height;
if (sk->numberOfSkeletonsDetected>0)
{
//------------------------------------------------------------------------------------
sk->skeletons[0].bbox2D.minimumX = sk->skeletons[0].body.bbox2D[0].x;
sk->skeletons[0].bbox2D.minimumY = sk->skeletons[0].body.bbox2D[0].y;
convertNormalized2DJointsToOriginalImageCoordinates(
jest,
&sk->skeletons[0].bbox2D.minimumX,
&sk->skeletons[0].bbox2D.minimumY,
0
);
//------------------------------------------------------------------------------------
sk->skeletons[0].bbox2D.maximumX = sk->skeletons[0].body.bbox2D[1].x;
sk->skeletons[0].bbox2D.maximumY = sk->skeletons[0].body.bbox2D[1].y;
convertNormalized2DJointsToOriginalImageCoordinates(
jest,
&sk->skeletons[0].bbox2D.maximumX,
&sk->skeletons[0].bbox2D.maximumY,
0
);
//------------------------------------------------------------------------------------
sk->skeletons[0].bbox2D.populated=1;
}
if (
getBestCropWindow(
tryForMaximumCrop,
&jest->crop.offsetX,
&jest->crop.offsetY,
&jest->crop.croppedDimensionWidth,
&jest->crop.croppedDimensionHeight,
&sk->skeletons[0].bbox2D,
jest->inputWidth2DJointDetector,
jest->inputHeight2DJointDetector,
frame.size().width,
frame.size().height
)
)
{
if (jest->crop.croppedDimensionWidth!=jest->crop.croppedDimensionHeight)
{
fprintf(stderr,"Bounding box produced was not a rectangle (%ux%u)..!\n",jest->crop.croppedDimensionWidth,jest->crop.croppedDimensionHeight);
}
cv::Rect rectangleROI(
jest->crop.offsetX,
jest->crop.offsetY,
jest->crop.croppedDimensionWidth,
jest->crop.croppedDimensionHeight
);
frame = frame(rectangleROI);
sk->skeletons[0].bbox2D.populated=0;
result=1;
}
cv::resize(frame,frame,cv::Size(jest->inputWidth2DJointDetector,jest->inputHeight2DJointDetector));
return result;
}
animation/MocapNET/src/JointEstimator2D/visualization.hpp
+31
View File
@@ -0,0 +1,31 @@
#pragma once
#include "opencv2/opencv.hpp"
/** @file webcam.cpp
* @brief This is a simple test file to make sure your camera or video files can be opened using OpenCV
* @author Ammar Qammaz (AmmarkoV)
*/
#include <stdio.h>
#include <vector>
#include "jointEstimator2D.hpp"
#include "../MocapNET2/MocapNETLib2/IO/commonSkeleton.hpp"
using namespace cv;
int visualizeHeatmaps(struct JointEstimator2D * jointEstimator,std::vector<std::vector<float> > heatmapNNOutput,unsigned int frameNumber);
void dj_drawExtractedSkeletons(
cv::Mat img,
struct Skeletons2DDetected * sk,
float factorX,
float factorY
);
int cropAndResizeCVMatToMatchSkeleton(
struct JointEstimator2D * jest,
cv::Mat & frame,
struct Skeletons2DDetected * sk
);
animation/MocapNET/src/JointEstimator2D/webcamAnd2DJoints.cpp
+157
View File
@@ -0,0 +1,157 @@
#include "opencv2/opencv.hpp"
/** @file webcam.cpp
* @brief This is a simple test file to make sure your camera or video files can be opened using OpenCV
* @author Ammar Qammaz (AmmarkoV)
*/
#include <stdio.h>
#include "cameraControl.hpp"
#include "jointEstimator2D.hpp"
#include "visualization.hpp"
#include "../MocapNET2/MocapNETLib2/IO/commonSkeleton.hpp"
using namespace cv;
int main(int argc, char *argv[])
{
unsigned int width = 640;
unsigned int height = 480;
const char * webcam = 0;
int heatmapDebugVisualizations=0;
//------------------------------------------------------
// Parse arguments
//------------------------------------------------------
for (int i=0; i<argc; i++)
{
if (strcmp(argv[i],"--from")==0)
{
if (argc>i+1)
{
webcam = argv[i+1];
}
} else
if (strcmp(argv[i],"--debug")==0)
{
heatmapDebugVisualizations = 1;
} else
if (strcmp(argv[i],"--size")==0)
{
if (argc>i+2)
{
width = atoi(argv[i+1]);
height = atoi(argv[i+2]);
}
}
}
//------------------------------------------------------
VideoCapture cap(webcam); // open the default camera
std::cerr<<"Trying to open webcam ("<<webcam<<" @ "<<width<<","<<height<<") \n";
if (!cap.isOpened()) // check if succeeded to connect to the camera
{
std::cerr<<"Openning camera failed\n";
return 1;
}
if (strstr(webcam,"/dev/video")!=0)
{
cap.set(cv::CAP_PROP_FRAME_WIDTH,width);
cap.set(cv::CAP_PROP_FRAME_HEIGHT,height);
}
struct JointEstimator2D jointEstimator;
if (loadJointEstimator2D(&jointEstimator,JOINT_2D_ESTIMATOR_FORTH,1,0))
{
cv::Mat viewMat = Mat(Size(jointEstimator.inputWidth2DJointDetector,jointEstimator.inputHeight2DJointDetector),CV_8UC3, Scalar(0,0,0));
struct Skeletons2DDetected result= {0};
//cv::namedWindow("Video Input Feed",1);
//cv::moveWindow("Video Input Feed",0,368);
cv::namedWindow("Skeletons",1);
cv::moveWindow("Skeletons",0,0);
unsigned int frameNumber=0;
unsigned long elapsedTime=0;
unsigned int brokenFrames = 0;
for(;;)
{
Mat frame;
cap >> frame;
if ( (frame.size().width>0) && (frame.size().height>0) )
{
if (
!cropAndResizeCVMatToMatchSkeleton(
&jointEstimator,
frame,
&result
)
)
{
fprintf(stderr,"Failed to crop input video\n");
}
//imshow("Video Input Feed", frame);
frame.copyTo(viewMat);
// viewMat.setTo(Scalar(0,0,0));
//This needs to be done to get tensorflow output..
frame.convertTo(frame,CV_32FC3);
//Keep time
unsigned long startTime = GetTickCountMicrosecondsJointEstimator();
std::vector<std::vector<float> > heatmaps = getHeatmaps(
&jointEstimator,
frame.data,
jointEstimator.inputWidth2DJointDetector,
jointEstimator.inputHeight2DJointDetector
);
if (heatmaps.size()>0)
{
if (heatmapDebugVisualizations)
{ visualizeHeatmaps(&jointEstimator,heatmaps,frameNumber); }
estimate2DSkeletonsFromHeatmaps(&jointEstimator,&result,heatmaps);
//Keep time
unsigned long endTime = GetTickCountMicrosecondsJointEstimator();
elapsedTime += endTime - startTime;
dj_drawExtractedSkeletons(
viewMat,
&result,
jointEstimator.inputWidth2DJointDetector,
jointEstimator.inputHeight2DJointDetector
);
imshow("Skeletons", viewMat);
}
++frameNumber;
}
else
{
std::cerr<<"Broken frame.. \n";
++brokenFrames;
}
waitKey(1);
if (brokenFrames>10) { break; }
}
if (frameNumber>0)
{
fprintf(stderr,"Elapsed time is %lu microsconds for %u frames\n",elapsedTime,frameNumber);
fprintf(stderr,"Framerate : %0.2f fps\n",(float) elapsedTime/(1000* frameNumber));
}
}
// the camera will be deinitialized automatically in VideoCapture destructor
return 0;
}