mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-16 00:13:07 +00:00
Add MocapNET
This commit is contained in:
Justin John
@@ -0,0 +1,748 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
"""
|
||||
Author : "Ammar Qammaz"
|
||||
Copyright : "2022 Foundation of Research and Technology, Computer Science Department Greece, See license.txt"
|
||||
License : "FORTH"
|
||||
"""
|
||||
#TODO : FIX 3D MODE!
|
||||
import os
|
||||
import sys
|
||||
import gc
|
||||
import time
|
||||
|
||||
#--------------------------------------------------
|
||||
import numpy as np
|
||||
#--------------------------------------------------
|
||||
from readCSV import parseConfiguration,checkIfConfigurationHierarchyIsTheSameAsLabelList,readCSVLabels,readCSVFile,checkIfTestAndTrainListsAreOk,readGroundTruthFile,splitNumpyArray
|
||||
from DNNModel import setupDNNModelsUsingJSONConfiguration
|
||||
from DNNTraining import printTrainingVersion,regularTraining,onlineHardExampleMiningTraining
|
||||
from tools import bcolors,dumpListToFile,getRAMInformation,createDirectory,checkIfFileExists,getConfigurationJointIsDeclaredInHierarchy,getConfigurationJointPriority,getParentNetwork,notification
|
||||
from getModelFromDatabase import retrieveAndSetupModel,retrieveAndSetupBasedOnSerial
|
||||
|
||||
print("evaluateMocapNET starting up..\n")
|
||||
printTrainingVersion()
|
||||
print("""
|
||||
███████╗██╗ ██╗ █████╗ ██╗ ██╗ ██╗ █████╗ ████████╗███████╗
|
||||
██╔════╝██║ ██║██╔══██╗██║ ██║ ██║██╔══██╗╚══██╔══╝██╔════╝
|
||||
█████╗ ██║ ██║███████║██║ ██║ ██║███████║ ██║ █████╗
|
||||
██╔══╝ ╚██╗ ██╔╝██╔══██║██║ ██║ ██║██╔══██║ ██║ ██╔══╝
|
||||
███████╗ ╚████╔╝ ██║ ██║███████╗╚██████╔╝██║ ██║ ██║ ███████╗
|
||||
╚══════╝ ╚═══╝ ╚═╝ ╚═╝╚══════╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝ ╚══════╝
|
||||
|
||||
""")
|
||||
|
||||
print("""
|
||||
██████╗ ██████╗ ██████╗ ██╗ ██╗
|
||||
██╔══██╗██╔═══██╗██╔══██╗╚██╗ ██╔╝
|
||||
██████╔╝██║ ██║██║ ██║ ╚████╔╝
|
||||
██╔══██╗██║ ██║██║ ██║ ╚██╔╝
|
||||
██████╔╝╚██████╔╝██████╔╝ ██║
|
||||
╚═════╝ ╚═════╝ ╚═════╝ ╚═╝
|
||||
""")
|
||||
|
||||
print("Ensuring the same seed for reproductible results always..\n")
|
||||
from numpy.random import seed
|
||||
seed(1)
|
||||
np.set_printoptions(precision=2)
|
||||
np.set_printoptions(formatter={"float_kind": lambda x: "%g" % x})
|
||||
|
||||
|
||||
def setBackendToHalfFloats():
|
||||
dtype='float16'
|
||||
print("\nUsing Half-Floats for training ");
|
||||
|
||||
|
||||
#-----------------------------------------------------------------------------------------------------------------------
|
||||
def identifyProblematicOutput(mapPath):
|
||||
print("READING evaluateMocapNET.lastresults.csv")
|
||||
#-----------------------------------------------------------------
|
||||
losses = readCSVFile("evaluateMocapNET.lastresults.csv" , 1.0 , 0)
|
||||
#-----------------------------------------------------------------
|
||||
f = open("evaluateMocapNET.identified.csv", "w")
|
||||
f.write("SampleID,MAE,BVH,fID,cumulative\n")
|
||||
# Using readlines()
|
||||
file1 = open(mapPath, 'r')
|
||||
Lines = file1.readlines()
|
||||
#-----------------------------------------------------------------
|
||||
minimum=np.min(losses['body'])
|
||||
maximum=np.max(losses['body'])
|
||||
median=np.median(losses['body'])
|
||||
mean=np.mean(losses['body'])
|
||||
std=np.std(losses['body'])
|
||||
var=np.var(losses['body'])
|
||||
print(" Min/Max=%0.2f/%0.2f,Median=%0.2f,Mean=%0.2f,Std=%0.2f,Var=%0.2f" % (minimum,maximum,median,mean,std,var))
|
||||
sampleID = 0
|
||||
hits = 0
|
||||
previousBVHFile="dummy initial value"
|
||||
consecutiveHits=0
|
||||
errorThreshold=250.0
|
||||
for item in losses['body']:
|
||||
if (item>errorThreshold):
|
||||
bvhFile = Lines[sampleID+1].split(",")[0]
|
||||
fID = int(Lines[sampleID+1].split(",")[1])
|
||||
#----------------------------------------------
|
||||
if (previousBVHFile!=bvhFile):
|
||||
consecutiveHits=0
|
||||
previousBVHFile=bvhFile
|
||||
else:
|
||||
consecutiveHits=consecutiveHits+1
|
||||
#----------------------------------------------
|
||||
print(sampleID," error=",item[0]," -> ",bvhFile," fID ",fID)
|
||||
f.write("%u,%f,%s,%u,%u\n"%(sampleID,item[0],bvhFile,fID,consecutiveHits))
|
||||
hits=hits+1
|
||||
sampleID=sampleID+1
|
||||
#---------------------------------------------------------------
|
||||
print("Found ",hits," occurances of error > ",errorThreshold)
|
||||
file1.close()
|
||||
|
||||
#-----------------------------------------------------------------------------------------------------------------------
|
||||
def appendRAWResultsForGNUplot(outputFile,globalJointDistances):
|
||||
fileHandler = open(outputFile, "w")
|
||||
fileHandler.write(outputFile)
|
||||
fileHandler.write("\n")
|
||||
for measurement in globalJointDistances:
|
||||
fileHandler.write(str(measurement))
|
||||
fileHandler.write("\n")
|
||||
fileHandler.close()
|
||||
#-----------------------------------------------------------------------------------------------------------------------
|
||||
def ground3DJointsFromBVH(bvh,rawBVHPrediction :dict):
|
||||
#Extract a BVH dict of BVH motion fields
|
||||
#Modify our BVH armature with the new BVH values
|
||||
if (bvh.modify(rawBVHPrediction)):
|
||||
|
||||
#Render to 2D/3D
|
||||
bvh.processFrame(0) #only have 1 frame ID
|
||||
|
||||
#Retreive 2D/3D Values
|
||||
output2D = dict()
|
||||
output3D = dict()
|
||||
for jointID in range(0,bvh.numberOfJoints):
|
||||
#-------------------------------------------
|
||||
jointName = bvh.getJointName(jointID).lower()
|
||||
#-------------------------------------------
|
||||
x3D,y3D,z3D = bvh.getJoint3D(jointID)
|
||||
output3D["3DX_"+jointName]=float(x3D)
|
||||
output3D["3DY_"+jointName]=float(y3D)
|
||||
output3D["3DZ_"+jointName]=float(z3D)
|
||||
#-------------------------------------------
|
||||
x2D,y2D = bvh.getJoint2D(jointID)
|
||||
output2D["2DX_"+jointName]=float(x2D)
|
||||
output2D["2DY_"+jointName]=float(y2D)
|
||||
#-------------------------------------------
|
||||
else:
|
||||
print(bcolors.FAIL,"We where unable to process the BVH output",bcolors.ENDC)
|
||||
|
||||
return output3D
|
||||
|
||||
|
||||
def evaluateMocapNET(useCSVBackend,groundTruthTrain,groundTruthTest):
|
||||
#----------------------------------------------------------------------------------------------------------
|
||||
#----------------------------------------------------------------------------------------------------------
|
||||
# Command-Line configuration and parameters and stuff
|
||||
#----------------------------------------------------------------------------------------------------------
|
||||
#----------------------------------------------------------------------------------------------------------
|
||||
#Note starting time
|
||||
startEverythingAt = time.time()
|
||||
|
||||
#1/0 = safe settings
|
||||
saveResultsToDisk=1
|
||||
if (not saveResultsToDisk):
|
||||
input("\n\n\n\n\nResults will not be saved..! All of the computing time will go in vain..\nAre you sure you want to do this ?\n\n\n\n")
|
||||
|
||||
#Options in case of cluster operation
|
||||
startPosition = 0
|
||||
endPosition = 0
|
||||
jobsPerCluster = 0
|
||||
jobsPerClusterStart = 0
|
||||
jobsPerClusterEnd = 0
|
||||
|
||||
#Cheapout on the test by default
|
||||
testMemPercentage=1.0
|
||||
|
||||
#Use the whole training Dataset by default
|
||||
memPercentage=1.0
|
||||
|
||||
#networkCompression=1.0
|
||||
useHalfFloats = 0
|
||||
useRadians = 0
|
||||
|
||||
#Resume training options
|
||||
step=1
|
||||
compareAllJoints=0
|
||||
|
||||
#AutoStart Tensorboard, disabled by default!
|
||||
autostartTensorboard=0
|
||||
|
||||
doDebugVisualization = False
|
||||
doProfiling = False
|
||||
doOnlyIdentification = False
|
||||
|
||||
#I/O settings
|
||||
outputDirectoryForTrainedModels = "step0FrontBody"
|
||||
dataFile = "body"
|
||||
hierarchyPartName = "body"
|
||||
outputMode = "bvh"
|
||||
|
||||
useTestData = True
|
||||
|
||||
|
||||
doNNEveryNFrames = 1
|
||||
doHCDPostProcessing = 0
|
||||
hcdLearningRate = 0.001
|
||||
hcdEpochs = 30
|
||||
hcdIterations = 15
|
||||
|
||||
#This needs to be populated using the --config argument otherwise session will fail..
|
||||
configuration = []
|
||||
configurationPath=""
|
||||
|
||||
engine = "onnx"
|
||||
label = "evaluation"
|
||||
doProcrustes = 1
|
||||
|
||||
# Sample call with some example parameters : ( needs --all body to have all joints ) if you see MocapNET always
|
||||
# gets initiallized with upper/lower networks, this is just needed to correctly parse the ground truth test files
|
||||
# --getPCA 15 105
|
||||
# python3 -m evaluateMocapNET --getPCA 15 32 --config dataset/body_configuration.json --skip 5 --all body
|
||||
# python3 -m evaluateMocapNET --config dataset/body_configuration.json --all body --skip 5 --mem 100
|
||||
# python3 -m evaluateMocapNET --getPCA 210 64 --mem 10 --config dataset/body_configuration.json --all body
|
||||
# python3 -m evaluateMocapNET --get 128 --config dataset/body_configuration.json --all body
|
||||
# python3 -m evaluateMocapNET --config dataset/body_configuration.json --all body --engine onnx --3d #<- Evaluate 3D data!
|
||||
#
|
||||
# To generate evaluateMocapNET.lastresults.csv and then evaluateMocapNET.identified.csv do :
|
||||
# python3 -m evaluateMocapNET --config dataset/body_configuration.json --all body --mem 1.0 --engine onnx --traindataset
|
||||
# python3 -m evaluateMocapNET --config dataset/body_configuration.json --all body --mem 10 --engine onnx --traindataset --identify
|
||||
#-----------------------------------------------------------------------------------------------------------------------------------------
|
||||
if (len(sys.argv)>1):
|
||||
print('Argument List:', str(sys.argv))
|
||||
for i in range(0, len(sys.argv)):
|
||||
if (sys.argv[i]=="--hcd"):
|
||||
doHCDPostProcessing = 1
|
||||
if (sys.argv[i]=="--visualize"):
|
||||
doDebugVisualization = True
|
||||
if (sys.argv[i]=="--ik"):
|
||||
doHCDPostProcessing = 1
|
||||
hcdLearningRate = float(sys.argv[i+1])
|
||||
hcdEpochs = int(sys.argv[i+2])
|
||||
hcdIterations = int(sys.argv[i+3])
|
||||
if (sys.argv[i]=="--nnsubsample"):
|
||||
doNNEveryNFrames = int(sys.argv[i+1])
|
||||
if (sys.argv[i]=="--noprocrustes"):
|
||||
doProcrustes=0
|
||||
if (sys.argv[i]=="--3d"):
|
||||
print(bcolors.WARNING,"Evaluate in 3D mode\n",bcolors.ENDC)
|
||||
#if needed by the model this will get auto activated later..
|
||||
outputMode = "3d"
|
||||
if (sys.argv[i]=="--identify"):
|
||||
doOnlyIdentification=True
|
||||
if (sys.argv[i]=="--engine"):
|
||||
engine=sys.argv[i+1]
|
||||
print("Selecting engine : ",engine)
|
||||
if (sys.argv[i]=="--profile"):
|
||||
doProfiling=True
|
||||
if (sys.argv[i]=="--label"):
|
||||
label=sys.argv[i+1]
|
||||
if (sys.argv[i]=="--get"):
|
||||
serial =int(sys.argv[i+1])
|
||||
retrieveAndSetupBasedOnSerial(serial,download=1)
|
||||
if (sys.argv[i]=="--getPCA"):
|
||||
PCADimensions =int(sys.argv[i+1])
|
||||
parameterCount=int(sys.argv[i+2])
|
||||
label = "evaluation_pca%u_param%u" % (PCADimensions,parameterCount)
|
||||
print("Get ",label)
|
||||
retrieveAndSetupModel(PCADimensions,parameterCount,download=1)
|
||||
if (sys.argv[i]=="--config"):
|
||||
configurationPath=sys.argv[i+1]
|
||||
configuration = parseConfiguration(configurationPath)
|
||||
setupDNNModelsUsingJSONConfiguration(configuration)
|
||||
if (sys.argv[i]=="--mem"):
|
||||
print("\nMemory usage ",sys.argv[i+1]);
|
||||
memPercentage=float(sys.argv[i+1])
|
||||
if (sys.argv[i]=="--skip"):
|
||||
print("\nSkipping frames ",sys.argv[i+1]);
|
||||
step=int(sys.argv[i+1])
|
||||
if (sys.argv[i]=="--full"):
|
||||
print("\nComparing All Joints : ",sys.argv[i+1]);
|
||||
compareAllJoints=int(sys.argv[i+1])
|
||||
#-----------------------------------------------
|
||||
# New 4way split
|
||||
#-----------------------------------------------
|
||||
if (sys.argv[i]=="--traindataset"):
|
||||
print("\n Using training data instead of test data");
|
||||
useTestData=False
|
||||
if (sys.argv[i]=="--dataset"):
|
||||
print("\nOverriding dataset ",dataFile," and using ",sys.argv[i+1]);
|
||||
dataFile=sys.argv[i+1]
|
||||
if (sys.argv[i]=="--all"):
|
||||
#Don't mix this --all with the step2_OrientatonClassifier.py --all
|
||||
hierarchyPartName=sys.argv[i+1]
|
||||
dataFile="%s_all" % hierarchyPartName
|
||||
outputDirectoryForTrainedModels="step0_%s" % dataFile
|
||||
if (sys.argv[i]=="--back"):
|
||||
hierarchyPartName=sys.argv[i+1]
|
||||
dataFile="%s_back" % hierarchyPartName
|
||||
outputDirectoryForTrainedModels="step0_%s" % dataFile
|
||||
if (sys.argv[i]=="--front"):
|
||||
hierarchyPartName=sys.argv[i+1]
|
||||
dataFile="%s_front" % hierarchyPartName
|
||||
outputDirectoryForTrainedModels="step0_%s" % dataFile
|
||||
if (sys.argv[i]=="--left"):
|
||||
hierarchyPartName=sys.argv[i+1]
|
||||
dataFile="%s_left" % hierarchyPartName
|
||||
outputDirectoryForTrainedModels="step0_%s" % dataFile
|
||||
if (sys.argv[i]=="--right"):
|
||||
hierarchyPartName=sys.argv[i+1]
|
||||
dataFile="%s_right" % hierarchyPartName
|
||||
outputDirectoryForTrainedModels="step0_%s" % dataFile
|
||||
#-----------------------------------------------
|
||||
|
||||
if (len(configuration)==0):
|
||||
print(bcolors.FAIL,"Evaluation cannot run without a configuration!",bcolors.ENDC)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
#Resolve PCA name.. :(
|
||||
x = configuration['doPCA'].split("$")
|
||||
if (len(x)>0):
|
||||
#If there is a $ character this is our place holder to autocomplete our dataFile
|
||||
print("PCA filename is resolved from ",configuration['doPCA'])
|
||||
print(" to ")
|
||||
configuration['doPCA']=x[0]+dataFile+".pca"
|
||||
print(configuration['doPCA'])
|
||||
|
||||
|
||||
|
||||
print("Creating directory for training models "+outputDirectoryForTrainedModels)
|
||||
createDirectory(outputDirectoryForTrainedModels)
|
||||
#----------------------------------------------------------------------------------------------------------
|
||||
#----------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
#The default compatibility setting is the BMVC2019 2channel NSDM, however nowadays we use NSRM
|
||||
numberOfChannelsPerNSDMElement=2
|
||||
if (configuration['NSDMAlsoUseAlignmentAngles']==1):
|
||||
numberOfChannelsPerNSDMElement=1
|
||||
print("Number of Channels Per NSDM element ",numberOfChannelsPerNSDMElement)
|
||||
configuration['numberOfChannelsPerNSDMElement']=numberOfChannelsPerNSDMElement
|
||||
|
||||
|
||||
#Printout the current configuration with regard to what network to inherit from
|
||||
if (configuration['rememberWeights']):
|
||||
print("Reloading previous training results")
|
||||
else:
|
||||
print("Not reloading")
|
||||
|
||||
|
||||
#----------------------------------------------------------------------------------------------------------
|
||||
#----------------------------------------------------------------------------------------------------------
|
||||
# Read our training and test datasets
|
||||
#----------------------------------------------------------------------------------------------------------
|
||||
#----------------------------------------------------------------------------------------------------------
|
||||
print("\n-------------------------------------------------");
|
||||
startAt = time.time()
|
||||
|
||||
#Special case if the user has a set memory requirement we try to mimic it for test as well..
|
||||
if (testMemPercentage==1.0):
|
||||
if (memPercentage==0): testMemPercentage=memPercentage
|
||||
if (memPercentage>1): testMemPercentage=memPercentage
|
||||
|
||||
#Hardcoded path for dataset if using CSV file
|
||||
#datasetDirectory="/home/ammar/Documents/Programming/DNNTracker/DNNTracker/dataset"
|
||||
#datasetDirectory="../../DNNTracker/dataset"
|
||||
datasetDirectory="dataset/generated/"
|
||||
|
||||
#Auto activate 3D outputMode if needed
|
||||
if ('outputMode' in configuration) and (configuration['outputMode'] == "3d"):
|
||||
outputMode="3d"
|
||||
|
||||
#Eigenposes should be loaded before Train/Test files since they are used in the input transform step
|
||||
#---------------------------------------------------------------------------------------------------
|
||||
if ("eigenPoses" in configuration):
|
||||
#We have eigenPoses Declared!
|
||||
if (configuration['eigenPoses']==1):
|
||||
#We have eigenPoses Active!
|
||||
configuration['eigenPoseData'] = readGroundTruthFile(
|
||||
configuration,
|
||||
"Eigenposes",
|
||||
"%s/2d_%s_eigenposes.csv" % (datasetDirectory,dataFile),
|
||||
"%s/%s_%s_eigenposes.csv" % (datasetDirectory,outputMode,dataFile), # comparisonMode / configuration['outputMode'] is either bvh or 3d
|
||||
1.0,
|
||||
numberOfChannelsPerNSDMElement,
|
||||
useRadians,
|
||||
useHalfFloats
|
||||
)
|
||||
|
||||
|
||||
datasetUsedLabel="Train"
|
||||
filenamePostfix=""
|
||||
if (useTestData):
|
||||
datasetUsedLabel="Test"
|
||||
filenamePostfix="_test"
|
||||
|
||||
#Easier and faster way to get output list..
|
||||
if (useCSVBackend):
|
||||
inputLabelsWithoutNSDM = readCSVLabels("%s/2d_%s%s.csv" % (datasetDirectory,dataFile,filenamePostfix))
|
||||
outputLabels = readCSVLabels("%s/%s_%s%s.csv" % (datasetDirectory,outputMode,dataFile,filenamePostfix)) #outputMode is either bvh or 3d
|
||||
mapPath = "%s/map_%s%s.csv" % (datasetDirectory,dataFile,filenamePostfix)
|
||||
else:
|
||||
inputLabelsWithoutNSDM = groundTruthTrain["labelInNoNSDM"]
|
||||
outputLabels = groundTruthTrain["labelOut"]
|
||||
#No Map Path will cause an error in --identify
|
||||
|
||||
|
||||
if (not checkIfConfigurationHierarchyIsTheSameAsLabelList(configuration,inputLabelsWithoutNSDM,0)):
|
||||
print(bcolors.WARNING +"step0_DNNTrainer.py : Inconsistent Configuration association compared to the inputLabelList" + bcolors.ENDC )
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
print("Input Labels (no NSDM) : ");
|
||||
print(inputLabelsWithoutNSDM)
|
||||
|
||||
print("\nOutput Labels : ");
|
||||
print(outputLabels)
|
||||
|
||||
numberOfOutputColumns=len(outputLabels)
|
||||
#Initially we would have to do all outputs ourself
|
||||
endPosition=numberOfOutputColumns
|
||||
|
||||
#But if a cluster has been set up we should only do *some* of the training
|
||||
if (jobsPerCluster):
|
||||
startPosition = jobsPerClusterStart
|
||||
endPosition = jobsPerClusterEnd
|
||||
|
||||
|
||||
print("\nWill run from ",startPosition," to ",endPosition," / ",numberOfOutputColumns);
|
||||
|
||||
RAMBefore = getRAMInformation()
|
||||
print("Initial | free RAM ",RAMBefore['free']," KB");
|
||||
|
||||
#If we want to use the CSV loading back end here
|
||||
#otherwise we assume groundTruthTrain and groundTruthTest have been already provided
|
||||
if (useCSVBackend):
|
||||
#---------------------------------------------------------------
|
||||
#Simplify..
|
||||
data2D = readCSVFile("%s/2d_%s%s.csv" % (datasetDirectory,dataFile,filenamePostfix),testMemPercentage,useHalfFloats)
|
||||
dataBVH = readCSVFile("%s/%s_%s%s.csv" % (datasetDirectory,outputMode,dataFile,filenamePostfix),testMemPercentage,useHalfFloats)
|
||||
groundTruthTest = dict()
|
||||
groundTruthTest['labelIn'] = data2D['label']
|
||||
groundTruthTest['in'] = data2D['body']
|
||||
groundTruthTest['labelOut'] = dataBVH['label']
|
||||
groundTruthTest['out'] = dataBVH['body']
|
||||
checkIfTestAndTrainListsAreOk(groundTruthTest,groundTruthTrain)
|
||||
#---------------------------------------------------------------
|
||||
|
||||
RAMAfter = getRAMInformation()
|
||||
print("After reading the dataset | free RAM ",RAMAfter['free']," KB");
|
||||
#----------------------------------------------------------------------------------------------------------
|
||||
endAt = time.time()
|
||||
print("Memory Occupied ",(RAMAfter['used']-RAMBefore['used'])/1024," MB")
|
||||
print("Time required to load from disk was ",(endAt-startAt)/60," mins")
|
||||
#
|
||||
print("\n-------------------------------------------------");
|
||||
#----------------------------------------------------------------------------------------------------------
|
||||
#----------------------------------------------------------------------------------------------------------
|
||||
|
||||
#We have lodaded everything as normal but we only want to do identification in this run!
|
||||
if (doOnlyIdentification):
|
||||
identifyProblematicOutput(mapPath)
|
||||
sys.exit(0)
|
||||
|
||||
#At this point we are ready..
|
||||
#We should free whatever memory we have allocated because it is training time..!
|
||||
#------------------------
|
||||
gc.collect()
|
||||
#------------------------
|
||||
|
||||
#Select a MocapNET class from tensorflow/tensorrt/onnx/tf-lite engines
|
||||
from MocapNET import easyMocapNETConstructor
|
||||
mnet = easyMocapNETConstructor(engine,
|
||||
doProfiling=doProfiling,
|
||||
doHCDPostProcessing = doHCDPostProcessing,
|
||||
hcdLearningRate = hcdLearningRate,
|
||||
hcdEpochs = hcdEpochs,
|
||||
hcdIterations = hcdIterations
|
||||
) #<- do not use post processing to actually evaluate MocapNET NN
|
||||
mnet.test()
|
||||
|
||||
from align3DPoints import compareGroundTruthToPrediction
|
||||
|
||||
# Use body to grab all joints
|
||||
#python3 evaluateMocapNET.py --mem 50 --config dataset/body_configuration.json --all upperbody
|
||||
#-----------------------------------------------
|
||||
totalSamples = 0
|
||||
totalError = 0.0
|
||||
averageErrorDistances = list()
|
||||
allErrorDistanceSamples = list()
|
||||
errorSamplesPerJoint = dict()
|
||||
#-----------------------------------------------
|
||||
numberOfSamples = len(groundTruthTest['in'])
|
||||
|
||||
#Declare joints we are interested in..
|
||||
#-----------------------------------------
|
||||
JOINTS_TO_COMPARE=list()
|
||||
if (compareAllJoints==0):
|
||||
JOINTS_TO_COMPARE.append('neck') #0
|
||||
JOINTS_TO_COMPARE.append('rshoulder') #1
|
||||
JOINTS_TO_COMPARE.append('relbow') #2
|
||||
JOINTS_TO_COMPARE.append('rhand') #3
|
||||
JOINTS_TO_COMPARE.append('lshoulder') #4
|
||||
JOINTS_TO_COMPARE.append('lelbow') #5
|
||||
JOINTS_TO_COMPARE.append('lhand') #6
|
||||
JOINTS_TO_COMPARE.append('hip') #7
|
||||
JOINTS_TO_COMPARE.append('rhip') #8
|
||||
JOINTS_TO_COMPARE.append('rknee') #9
|
||||
JOINTS_TO_COMPARE.append('rfoot') #10
|
||||
JOINTS_TO_COMPARE.append('lhip') #11
|
||||
JOINTS_TO_COMPARE.append('lknee') #12
|
||||
JOINTS_TO_COMPARE.append('lfoot') #13
|
||||
else:
|
||||
print(bcolors.FAIL,"TODO: IMPLEMENT COMPARE ALL JOINTS",bcolors.ENDC)
|
||||
sys.exit(1)
|
||||
|
||||
#Allocate errors per joint
|
||||
#-----------------------------------------
|
||||
if (len(JOINTS_TO_COMPARE)==0):
|
||||
numberOfJoints = len(configuration["hierarchy"])
|
||||
for jID in range (0,numberOfJoints):
|
||||
jointName = configuration["hierarchy"][jID]["joint"].lower()
|
||||
errorSamplesPerJoint[jointName]=list()
|
||||
JOINTS_TO_COMPARE.append(jointName)
|
||||
else:
|
||||
numberOfJoints = len(JOINTS_TO_COMPARE)
|
||||
for jID in range (0,numberOfJoints):
|
||||
jointName = JOINTS_TO_COMPARE[jID].lower()
|
||||
errorSamplesPerJoint[jointName]=list()
|
||||
#-----------------------------------------
|
||||
print("Doing evaluation based on errors of the following ",numberOfJoints," joints : ",JOINTS_TO_COMPARE)
|
||||
|
||||
|
||||
f = open("evaluateMocapNET.lastresults.csv", "w")
|
||||
#Append CSV Log
|
||||
f.write("mae_from_%u_to_%u_step_%u\n" % (0,numberOfSamples,step))
|
||||
|
||||
framerates = list()
|
||||
beginTime = time.time()
|
||||
evaluationMode = "?"
|
||||
frameNumber = 0 #<- simpler counter for nn subsampling
|
||||
for thisSample in range(0,numberOfSamples,step) :
|
||||
thisInputRaw = groundTruthTest['in'][thisSample]
|
||||
correctOutputRaw = groundTruthTest['out'][thisSample]
|
||||
|
||||
regressedResult = dict()
|
||||
correctResult = dict()
|
||||
|
||||
|
||||
#DATA Looks like this : { 'labelInNoNSDM':names2D, 'labelIn':inputLabels, 'in':npInput, 'labelOut':outputLabels, 'out':npOutput };
|
||||
thisInput = dict()
|
||||
for i in range(0,len(groundTruthTest['labelIn'])):
|
||||
thisLabelIn = groundTruthTest['labelIn'][i].lower()
|
||||
if ( ( thisLabelIn.find("edm-")== -1 ) and ( thisLabelIn.find("nsrm-")== -1 ) ):
|
||||
thisInput[thisLabelIn] = float(groundTruthTest['in'][thisSample][i])
|
||||
|
||||
#If we are dealing with just BVH output use training BVH output to extract 3D data
|
||||
#------------------------------------------------------------------------------------------------------
|
||||
if outputMode=="bvh":# (not 'outputMode' in configuration) or (configuration['outputMode'] == "bvh"):
|
||||
evaluationMode="bvh"
|
||||
correctOutput = dict()
|
||||
for i in range(0,len(groundTruthTest['labelOut'])):
|
||||
correctOutput[groundTruthTest['labelOut'][i]] = float(groundTruthTest['out'][thisSample][i])
|
||||
correctResult = ground3DJointsFromBVH(mnet.bvh,correctOutput)
|
||||
#------------------------------------------------------------------------------------------------------
|
||||
#If we are dealing with 3D output immediately use training 3D output to compare 3D data
|
||||
elif outputMode=="3d":# ('outputMode' in configuration) and (configuration['outputMode'] == "3d"):
|
||||
evaluationMode="3d"
|
||||
correctResult = dict()
|
||||
for i in range(0,len(groundTruthTest['labelOut'])):
|
||||
correctResult[groundTruthTest['labelOut'][i]] = float(groundTruthTest['out'][thisSample][i])
|
||||
#------------------------------------------------------------------------------------------------------
|
||||
startTime = time.time()
|
||||
#================================================
|
||||
#================================================
|
||||
doNN = 1
|
||||
if (doHCDPostProcessing) and (doNNEveryNFrames>1):
|
||||
doNN = (frameNumber%doNNEveryNFrames)==0
|
||||
#print("Frame ",frameNumber," doNN = ",doNN)
|
||||
regressedResult = mnet.predict3DJoints(thisInput,runNN=doNN)
|
||||
#================================================
|
||||
#================================================
|
||||
endTime = time.time()
|
||||
processing_time = endTime - startTime
|
||||
framerates.append(float(processing_time))
|
||||
if (processing_time==0):
|
||||
processing_time=1
|
||||
#------------------------------------------------------------------------------------------------------
|
||||
|
||||
#------------------------------------------------------------------------------------------------------
|
||||
#------------------------------------------------------------------------------------------------------
|
||||
#------------------------------------------------------------------------------------------------------
|
||||
# Debug visualization (we shouldn't be doing a visualization in evaluateMocapNET.py)
|
||||
#------------------------------------------------------------------------------------------------------
|
||||
if (doDebugVisualization):
|
||||
import cv2
|
||||
import numpy as np
|
||||
bgcolor = 30
|
||||
annotated_image = np.full((1080,1920,3), bgcolor , dtype=np.uint8) #<--
|
||||
from MocapNETVisualization import visualizeMocapNETEnsemble
|
||||
image,plotImage = visualizeMocapNETEnsemble(mnet,annotated_image)
|
||||
|
||||
cv2.imshow('MocapNET 4 Evaluation', image)
|
||||
if cv2.waitKey(1) & 0xFF == 27:
|
||||
break
|
||||
#------------------------------------------------------------------------------------------------------
|
||||
#------------------------------------------------------------------------------------------------------
|
||||
#------------------------------------------------------------------------------------------------------
|
||||
|
||||
#print("\n\n\n\nthisInput ",thisInput)
|
||||
#print("\n\n\n\ncorrectOutput ",correctOutput)
|
||||
#print("\n\n\n\regressedResult ",regressedResult)
|
||||
#print("\n\n\n\n")
|
||||
stats = compareGroundTruthToPrediction(
|
||||
configuration,
|
||||
correctResult, #<- We want to compare this
|
||||
regressedResult, #<- to This
|
||||
doProcrustes=doProcrustes,
|
||||
allowProcrustesToChangeScale=1,
|
||||
jointsToCompare=JOINTS_TO_COMPARE
|
||||
)
|
||||
|
||||
#Append CSV log
|
||||
f.write("%f\n" % (float(stats["meanAverageError"])))
|
||||
|
||||
#print("\nstats ",stats)
|
||||
print("\nstats={",end="")
|
||||
for k in stats.keys():
|
||||
print("'%s':%0.2f, "%(k,stats[k]),end="")
|
||||
print("}\n")
|
||||
|
||||
|
||||
progressMessage = "Progress : %0.2f %% %u / %u %0.2f fps" % ((100*thisSample)/numberOfSamples,thisSample,numberOfSamples,1/processing_time)
|
||||
print(progressMessage," Evaluation Mode :",bcolors.OKGREEN,evaluationMode,bcolors.ENDC,"\n\n\n\n")
|
||||
|
||||
#numberOfJoints = len(configuration["hierarchy"])
|
||||
for jID in range (0,numberOfJoints):
|
||||
jointName = JOINTS_TO_COMPARE[jID]
|
||||
if jointName in stats:
|
||||
allErrorDistanceSamples.append(float(stats[jointName]))
|
||||
errorSamplesPerJoint[jointName].append(float(stats[jointName]))
|
||||
totalSamples = totalSamples + 1
|
||||
totalError = totalError + float(stats[jointName])
|
||||
else:
|
||||
print("Could not find ",jointName)
|
||||
|
||||
averageErrorDistances.append(float(stats["meanAverageError"]))
|
||||
frameNumber = frameNumber + 1 #<- simpler counter for nn subsampling
|
||||
|
||||
endTime = time.time()
|
||||
print("Finished in ",(endTime-beginTime)/60," minutes")
|
||||
|
||||
f.close()
|
||||
|
||||
|
||||
import numpy as np
|
||||
appendRAWResultsForGNUplot("alldistance.raw",averageErrorDistances)
|
||||
#------------------------------------------
|
||||
for jID in range (0,numberOfJoints):
|
||||
jointName = JOINTS_TO_COMPARE[jID]
|
||||
appendRAWResultsForGNUplot("joint_%s.raw"%jointName,errorSamplesPerJoint[jointName])
|
||||
#------------------------------------------
|
||||
meanProcessingTime = np.mean(framerates)
|
||||
minimum = np.min(averageErrorDistances)
|
||||
maximum = np.max(averageErrorDistances)
|
||||
median = np.median(averageErrorDistances)
|
||||
mean = np.mean(averageErrorDistances)
|
||||
std = np.std(averageErrorDistances)
|
||||
var = np.var(averageErrorDistances)
|
||||
#------------------------------------------
|
||||
title_string=" Min/Max=%0.1f/%0.1f,Mean=%0.1f,Median=%0.1f,Std=%0.1f,Var=%0.1f" % (minimum,maximum,mean,median,std,var)
|
||||
|
||||
#Show a summary
|
||||
#------------------------------------------
|
||||
print("Flops : ",mnet.getModelFlops()," Model Params : ",mnet.getModelParameters())
|
||||
print("Upperbody Decomposition : ",mnet.ensemble["upperbody"].configuration["decompositionType"]," | Input Size : ",mnet.ensemble["upperbody"].configuration["PCADimensionsKept"])
|
||||
print("EDM upperbody : ",mnet.ensemble["upperbody"].configuration["EDM"])
|
||||
print("eNSRM upperbody : ",mnet.ensemble["upperbody"].configuration["eNSRM"])
|
||||
print("Lowerbody Decomposition : ",mnet.ensemble["lowerbody"].configuration["decompositionType"]," | Input Size : ",mnet.ensemble["lowerbody"].configuration["PCADimensionsKept"])
|
||||
print("EDM lowerbody : ",mnet.ensemble["lowerbody"].configuration["EDM"])
|
||||
print("eNSRM lowerbody : ",mnet.ensemble["lowerbody"].configuration["eNSRM"])
|
||||
print("Sample Step : %u / Total Samples %u "%(step,numberOfSamples))
|
||||
print("Procrustes : %u"%doProcrustes)
|
||||
print("Mean Framerate : %0.2f fps"%(1/meanProcessingTime))
|
||||
print("doHCDPostProcessing = ",doHCDPostProcessing)
|
||||
print("doNNEveryNFrames = ",doNNEveryNFrames)
|
||||
print("hcdLearningRate = ",hcdLearningRate)
|
||||
print("hcdEpochs = ",hcdEpochs)
|
||||
print("hcdIterations = ",hcdIterations)
|
||||
|
||||
print("Result : %s "%title_string)
|
||||
notification("MocapNET Network Evaluation","Done evaluating -> %s " % (title_string))
|
||||
|
||||
f = open(label+"_results.csv", "w")
|
||||
f.write("Params,PCA,FPS,Min,Max,Median,Mean,Std,Var,HCD,NNSubsample,HCD_LR,HCD_Epochs,HCD_Iterations\n")
|
||||
f.write("%0.2f"%(mnet.getModelParameters()/1000.0))
|
||||
f.write(",")
|
||||
f.write(str(mnet.ensemble["upperbody"].configuration["PCADimensionsKept"]))
|
||||
f.write(",")
|
||||
f.write("%0.2f"%(1/meanProcessingTime))
|
||||
f.write(",")
|
||||
f.write(str(minimum))
|
||||
f.write(",")
|
||||
f.write(str(maximum))
|
||||
f.write(",")
|
||||
f.write(str(median))
|
||||
f.write(",")
|
||||
f.write(str(mean))
|
||||
f.write(",")
|
||||
f.write(str(std))
|
||||
f.write(",")
|
||||
f.write(str(var))
|
||||
f.write(",")
|
||||
f.write(str(doHCDPostProcessing))
|
||||
f.write(",")
|
||||
f.write(str(doNNEveryNFrames))
|
||||
f.write(",")
|
||||
f.write(str(hcdLearningRate))
|
||||
f.write(",")
|
||||
f.write(str(hcdEpochs))
|
||||
f.write(",")
|
||||
f.write(str(hcdIterations))
|
||||
f.write("\n")
|
||||
f.close()
|
||||
|
||||
#Plotting !
|
||||
#------------------------------------------
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
fig, ax = plt.subplots(figsize=(8, 4))
|
||||
#------------------------------------------
|
||||
x = averageErrorDistances
|
||||
mu = 200
|
||||
sigma = 25
|
||||
n_bins = 100
|
||||
# plot the cumulative histogram
|
||||
n, bins, patches = ax.hist(x, n_bins, density=True, histtype='step',cumulative=True, label='Accuracy')
|
||||
#------------------------------------------
|
||||
# Add a line showing the expected distribution.
|
||||
y = ((1 / (np.sqrt(2 * np.pi) * sigma)) *
|
||||
np.exp(-0.5 * (1 / sigma * (bins - mu))**2))
|
||||
y = y.cumsum()
|
||||
y /= y[-1]
|
||||
#------------------------------------------
|
||||
#ax.plot(bins, y, 'k--', linewidth=1.5, label='Theoretical')
|
||||
# tidy up the figure
|
||||
ax.grid(True)
|
||||
ax.legend(loc='right')
|
||||
ax.set_title('Cumulative step histograms')
|
||||
ax.set_xlabel('3D Error (mm)')
|
||||
ax.set_ylabel('Likelihood of occurrence')
|
||||
#------------------------------------------
|
||||
#plt.show()
|
||||
print("Done")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
#When working standalone assume simple csv backend
|
||||
emptyData = dict()
|
||||
emptyTestData = { 'labelInNoNSDM': [], 'labelIn': [], 'in': [], 'labelOut':[], 'out': [] }
|
||||
useCSVBackend=1
|
||||
evaluateMocapNET(useCSVBackend,emptyData,emptyTestData)
|
||||
Reference in New Issue
Block a user