storytold/storyteller-ml
2
0
Fork 0
mirror of https://github.com/storytold/storyteller-ml.git synced 2026-10-16 00:13:07 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
65e8934d5ecd46ff27ddf5341ddfb73266b0bf1c
storyteller-ml/animation/MocapNET/src/python/mnet4/evaluateMocapNET.py
T
Justin John 65e8934d5e Add MocapNET
2023-12-15 04:22:41 +05:30

749 lines
36 KiB
Python
Raw Blame History

#!/usr/bin/python3
"""
Author : "Ammar Qammaz"
Copyright : "2022 Foundation of Research and Technology, Computer Science Department Greece, See license.txt"
License : "FORTH"
"""
#TODO : FIX 3D MODE!
import os
import sys
import gc
import time
#--------------------------------------------------
import numpy as np
#--------------------------------------------------
from readCSV import parseConfiguration,checkIfConfigurationHierarchyIsTheSameAsLabelList,readCSVLabels,readCSVFile,checkIfTestAndTrainListsAreOk,readGroundTruthFile,splitNumpyArray
from DNNModel import setupDNNModelsUsingJSONConfiguration
from DNNTraining import printTrainingVersion,regularTraining,onlineHardExampleMiningTraining
from tools import bcolors,dumpListToFile,getRAMInformation,createDirectory,checkIfFileExists,getConfigurationJointIsDeclaredInHierarchy,getConfigurationJointPriority,getParentNetwork,notification
from getModelFromDatabase import retrieveAndSetupModel,retrieveAndSetupBasedOnSerial
print("evaluateMocapNET starting up..\n")
printTrainingVersion()
print("""
███████╗██╗ ██╗ █████╗ ██╗ ██╗ ██╗ █████╗ ████████╗███████╗
██╔════╝██║ ██║██╔══██╗██║ ██║ ██║██╔══██╗╚══██╔══╝██╔════╝
█████╗ ██║ ██║███████║██║ ██║ ██║███████║ ██║ █████╗
██╔══╝ ╚██╗ ██╔╝██╔══██║██║ ██║ ██║██╔══██║ ██║ ██╔══╝
███████╗ ╚████╔╝ ██║ ██║███████╗╚██████╔╝██║ ██║ ██║ ███████╗
╚══════╝ ╚═══╝ ╚═╝ ╚═╝╚══════╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝ ╚══════╝
""")
print("""
██████╗ ██████╗ ██████╗ ██╗ ██╗
██╔══██╗██╔═══██╗██╔══██╗╚██╗ ██╔╝
██████╔╝██║ ██║██║ ██║ ╚████╔╝
██╔══██╗██║ ██║██║ ██║ ╚██╔╝
██████╔╝╚██████╔╝██████╔╝ ██║
╚═════╝ ╚═════╝ ╚═════╝ ╚═╝
""")
print("Ensuring the same seed for reproductible results always..\n")
from numpy.random import seed
seed(1)
np.set_printoptions(precision=2)
np.set_printoptions(formatter={"float_kind": lambda x: "%g" % x})
def setBackendToHalfFloats():
dtype='float16'
print("\nUsing Half-Floats for training ");
#-----------------------------------------------------------------------------------------------------------------------
def identifyProblematicOutput(mapPath):
print("READING evaluateMocapNET.lastresults.csv")
#-----------------------------------------------------------------
losses = readCSVFile("evaluateMocapNET.lastresults.csv" , 1.0 , 0)
#-----------------------------------------------------------------
f = open("evaluateMocapNET.identified.csv", "w")
f.write("SampleID,MAE,BVH,fID,cumulative\n")
# Using readlines()
file1 = open(mapPath, 'r')
Lines = file1.readlines()
#-----------------------------------------------------------------
minimum=np.min(losses['body'])
maximum=np.max(losses['body'])
median=np.median(losses['body'])
mean=np.mean(losses['body'])
std=np.std(losses['body'])
var=np.var(losses['body'])
print(" Min/Max=%0.2f/%0.2f,Median=%0.2f,Mean=%0.2f,Std=%0.2f,Var=%0.2f" % (minimum,maximum,median,mean,std,var))
sampleID = 0
hits = 0
previousBVHFile="dummy initial value"
consecutiveHits=0
errorThreshold=250.0
for item in losses['body']:
if (item>errorThreshold):
bvhFile = Lines[sampleID+1].split(",")[0]
fID = int(Lines[sampleID+1].split(",")[1])
#----------------------------------------------
if (previousBVHFile!=bvhFile):
consecutiveHits=0
previousBVHFile=bvhFile
else:
consecutiveHits=consecutiveHits+1
#----------------------------------------------
print(sampleID," error=",item[0]," -> ",bvhFile," fID ",fID)
f.write("%u,%f,%s,%u,%u\n"%(sampleID,item[0],bvhFile,fID,consecutiveHits))
hits=hits+1
sampleID=sampleID+1
#---------------------------------------------------------------
print("Found ",hits," occurances of error > ",errorThreshold)
file1.close()
#-----------------------------------------------------------------------------------------------------------------------
def appendRAWResultsForGNUplot(outputFile,globalJointDistances):
fileHandler = open(outputFile, "w")
fileHandler.write(outputFile)
fileHandler.write("\n")
for measurement in globalJointDistances:
fileHandler.write(str(measurement))
fileHandler.write("\n")
fileHandler.close()
#-----------------------------------------------------------------------------------------------------------------------
def ground3DJointsFromBVH(bvh,rawBVHPrediction :dict):
#Extract a BVH dict of BVH motion fields
#Modify our BVH armature with the new BVH values
if (bvh.modify(rawBVHPrediction)):
#Render to 2D/3D
bvh.processFrame(0) #only have 1 frame ID
#Retreive 2D/3D Values
output2D = dict()
output3D = dict()
for jointID in range(0,bvh.numberOfJoints):
#-------------------------------------------
jointName = bvh.getJointName(jointID).lower()
#-------------------------------------------
x3D,y3D,z3D = bvh.getJoint3D(jointID)
output3D["3DX_"+jointName]=float(x3D)
output3D["3DY_"+jointName]=float(y3D)
output3D["3DZ_"+jointName]=float(z3D)
#-------------------------------------------
x2D,y2D = bvh.getJoint2D(jointID)
output2D["2DX_"+jointName]=float(x2D)
output2D["2DY_"+jointName]=float(y2D)
#-------------------------------------------
else:
print(bcolors.FAIL,"We where unable to process the BVH output",bcolors.ENDC)
return output3D
def evaluateMocapNET(useCSVBackend,groundTruthTrain,groundTruthTest):
#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------
# Command-Line configuration and parameters and stuff
#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------
#Note starting time
startEverythingAt = time.time()
#1/0 = safe settings
saveResultsToDisk=1
if (not saveResultsToDisk):
input("\n\n\n\n\nResults will not be saved..! All of the computing time will go in vain..\nAre you sure you want to do this ?\n\n\n\n")
#Options in case of cluster operation
startPosition = 0
endPosition = 0
jobsPerCluster = 0
jobsPerClusterStart = 0
jobsPerClusterEnd = 0
#Cheapout on the test by default
testMemPercentage=1.0
#Use the whole training Dataset by default
memPercentage=1.0
#networkCompression=1.0
useHalfFloats = 0
useRadians = 0
#Resume training options
step=1
compareAllJoints=0
#AutoStart Tensorboard, disabled by default!
autostartTensorboard=0
doDebugVisualization = False
doProfiling = False
doOnlyIdentification = False
#I/O settings
outputDirectoryForTrainedModels = "step0FrontBody"
dataFile = "body"
hierarchyPartName = "body"
outputMode = "bvh"
useTestData = True
doNNEveryNFrames = 1
doHCDPostProcessing = 0
hcdLearningRate = 0.001
hcdEpochs = 30
hcdIterations = 15
#This needs to be populated using the --config argument otherwise session will fail..
configuration = []
configurationPath=""
engine = "onnx"
label = "evaluation"
doProcrustes = 1
# Sample call with some example parameters : ( needs --all body to have all joints ) if you see MocapNET always
# gets initiallized with upper/lower networks, this is just needed to correctly parse the ground truth test files
# --getPCA 15 105
# python3 -m evaluateMocapNET --getPCA 15 32 --config dataset/body_configuration.json --skip 5 --all body
# python3 -m evaluateMocapNET --config dataset/body_configuration.json --all body --skip 5 --mem 100
# python3 -m evaluateMocapNET --getPCA 210 64 --mem 10 --config dataset/body_configuration.json --all body
# python3 -m evaluateMocapNET --get 128 --config dataset/body_configuration.json --all body
# python3 -m evaluateMocapNET --config dataset/body_configuration.json --all body --engine onnx --3d #<- Evaluate 3D data!
#
# To generate evaluateMocapNET.lastresults.csv and then evaluateMocapNET.identified.csv do :
# python3 -m evaluateMocapNET --config dataset/body_configuration.json --all body --mem 1.0 --engine onnx --traindataset
# python3 -m evaluateMocapNET --config dataset/body_configuration.json --all body --mem 10 --engine onnx --traindataset --identify
#-----------------------------------------------------------------------------------------------------------------------------------------
if (len(sys.argv)>1):
print('Argument List:', str(sys.argv))
for i in range(0, len(sys.argv)):
if (sys.argv[i]=="--hcd"):
doHCDPostProcessing = 1
if (sys.argv[i]=="--visualize"):
doDebugVisualization = True
if (sys.argv[i]=="--ik"):
doHCDPostProcessing = 1
hcdLearningRate = float(sys.argv[i+1])
hcdEpochs = int(sys.argv[i+2])
hcdIterations = int(sys.argv[i+3])
if (sys.argv[i]=="--nnsubsample"):
doNNEveryNFrames = int(sys.argv[i+1])
if (sys.argv[i]=="--noprocrustes"):
doProcrustes=0
if (sys.argv[i]=="--3d"):
print(bcolors.WARNING,"Evaluate in 3D mode\n",bcolors.ENDC)
#if needed by the model this will get auto activated later..
outputMode = "3d"
if (sys.argv[i]=="--identify"):
doOnlyIdentification=True
if (sys.argv[i]=="--engine"):
engine=sys.argv[i+1]
print("Selecting engine : ",engine)
if (sys.argv[i]=="--profile"):
doProfiling=True
if (sys.argv[i]=="--label"):
label=sys.argv[i+1]
if (sys.argv[i]=="--get"):
serial =int(sys.argv[i+1])
retrieveAndSetupBasedOnSerial(serial,download=1)
if (sys.argv[i]=="--getPCA"):
PCADimensions =int(sys.argv[i+1])
parameterCount=int(sys.argv[i+2])
label = "evaluation_pca%u_param%u" % (PCADimensions,parameterCount)
print("Get ",label)
retrieveAndSetupModel(PCADimensions,parameterCount,download=1)
if (sys.argv[i]=="--config"):
configurationPath=sys.argv[i+1]
configuration = parseConfiguration(configurationPath)
setupDNNModelsUsingJSONConfiguration(configuration)
if (sys.argv[i]=="--mem"):
print("\nMemory usage ",sys.argv[i+1]);
memPercentage=float(sys.argv[i+1])
if (sys.argv[i]=="--skip"):
print("\nSkipping frames ",sys.argv[i+1]);
step=int(sys.argv[i+1])
if (sys.argv[i]=="--full"):
print("\nComparing All Joints : ",sys.argv[i+1]);
compareAllJoints=int(sys.argv[i+1])
#-----------------------------------------------
# New 4way split
#-----------------------------------------------
if (sys.argv[i]=="--traindataset"):
print("\n Using training data instead of test data");
useTestData=False
if (sys.argv[i]=="--dataset"):
print("\nOverriding dataset ",dataFile," and using ",sys.argv[i+1]);
dataFile=sys.argv[i+1]
if (sys.argv[i]=="--all"):
#Don't mix this --all with the step2_OrientatonClassifier.py --all
hierarchyPartName=sys.argv[i+1]
dataFile="%s_all" % hierarchyPartName
outputDirectoryForTrainedModels="step0_%s" % dataFile
if (sys.argv[i]=="--back"):
hierarchyPartName=sys.argv[i+1]
dataFile="%s_back" % hierarchyPartName
outputDirectoryForTrainedModels="step0_%s" % dataFile
if (sys.argv[i]=="--front"):
hierarchyPartName=sys.argv[i+1]
dataFile="%s_front" % hierarchyPartName
outputDirectoryForTrainedModels="step0_%s" % dataFile
if (sys.argv[i]=="--left"):
hierarchyPartName=sys.argv[i+1]
dataFile="%s_left" % hierarchyPartName
outputDirectoryForTrainedModels="step0_%s" % dataFile
if (sys.argv[i]=="--right"):
hierarchyPartName=sys.argv[i+1]
dataFile="%s_right" % hierarchyPartName
outputDirectoryForTrainedModels="step0_%s" % dataFile
#-----------------------------------------------
if (len(configuration)==0):
print(bcolors.FAIL,"Evaluation cannot run without a configuration!",bcolors.ENDC)
sys.exit(0)
#Resolve PCA name.. :(
x = configuration['doPCA'].split("$")
if (len(x)>0):
#If there is a $ character this is our place holder to autocomplete our dataFile
print("PCA filename is resolved from ",configuration['doPCA'])
print(" to ")
configuration['doPCA']=x[0]+dataFile+".pca"
print(configuration['doPCA'])
print("Creating directory for training models "+outputDirectoryForTrainedModels)
createDirectory(outputDirectoryForTrainedModels)
#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------
#The default compatibility setting is the BMVC2019 2channel NSDM, however nowadays we use NSRM
numberOfChannelsPerNSDMElement=2
if (configuration['NSDMAlsoUseAlignmentAngles']==1):
numberOfChannelsPerNSDMElement=1
print("Number of Channels Per NSDM element ",numberOfChannelsPerNSDMElement)
configuration['numberOfChannelsPerNSDMElement']=numberOfChannelsPerNSDMElement
#Printout the current configuration with regard to what network to inherit from
if (configuration['rememberWeights']):
print("Reloading previous training results")
else:
print("Not reloading")
#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------
# Read our training and test datasets
#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------
print("\n-------------------------------------------------");
startAt = time.time()
#Special case if the user has a set memory requirement we try to mimic it for test as well..
if (testMemPercentage==1.0):
if (memPercentage==0): testMemPercentage=memPercentage
if (memPercentage>1): testMemPercentage=memPercentage
#Hardcoded path for dataset if using CSV file
#datasetDirectory="/home/ammar/Documents/Programming/DNNTracker/DNNTracker/dataset"
#datasetDirectory="../../DNNTracker/dataset"
datasetDirectory="dataset/generated/"
#Auto activate 3D outputMode if needed
if ('outputMode' in configuration) and (configuration['outputMode'] == "3d"):
outputMode="3d"
#Eigenposes should be loaded before Train/Test files since they are used in the input transform step
#---------------------------------------------------------------------------------------------------
if ("eigenPoses" in configuration):
#We have eigenPoses Declared!
if (configuration['eigenPoses']==1):
#We have eigenPoses Active!
configuration['eigenPoseData'] = readGroundTruthFile(
configuration,
"Eigenposes",
"%s/2d_%s_eigenposes.csv" % (datasetDirectory,dataFile),
"%s/%s_%s_eigenposes.csv" % (datasetDirectory,outputMode,dataFile), # comparisonMode / configuration['outputMode'] is either bvh or 3d
1.0,
numberOfChannelsPerNSDMElement,
useRadians,
useHalfFloats
)
datasetUsedLabel="Train"
filenamePostfix=""
if (useTestData):
datasetUsedLabel="Test"
filenamePostfix="_test"
#Easier and faster way to get output list..
if (useCSVBackend):
inputLabelsWithoutNSDM = readCSVLabels("%s/2d_%s%s.csv" % (datasetDirectory,dataFile,filenamePostfix))
outputLabels = readCSVLabels("%s/%s_%s%s.csv" % (datasetDirectory,outputMode,dataFile,filenamePostfix)) #outputMode is either bvh or 3d
mapPath = "%s/map_%s%s.csv" % (datasetDirectory,dataFile,filenamePostfix)
else:
inputLabelsWithoutNSDM = groundTruthTrain["labelInNoNSDM"]
outputLabels = groundTruthTrain["labelOut"]
#No Map Path will cause an error in --identify
if (not checkIfConfigurationHierarchyIsTheSameAsLabelList(configuration,inputLabelsWithoutNSDM,0)):
print(bcolors.WARNING +"step0_DNNTrainer.py : Inconsistent Configuration association compared to the inputLabelList" + bcolors.ENDC )
sys.exit(1)
print("Input Labels (no NSDM) : ");
print(inputLabelsWithoutNSDM)
print("\nOutput Labels : ");
print(outputLabels)
numberOfOutputColumns=len(outputLabels)
#Initially we would have to do all outputs ourself
endPosition=numberOfOutputColumns
#But if a cluster has been set up we should only do *some* of the training
if (jobsPerCluster):
startPosition = jobsPerClusterStart
endPosition = jobsPerClusterEnd
print("\nWill run from ",startPosition," to ",endPosition," / ",numberOfOutputColumns);
RAMBefore = getRAMInformation()
print("Initial | free RAM ",RAMBefore['free']," KB");
#If we want to use the CSV loading back end here
#otherwise we assume groundTruthTrain and groundTruthTest have been already provided
if (useCSVBackend):
#---------------------------------------------------------------
#Simplify..
data2D = readCSVFile("%s/2d_%s%s.csv" % (datasetDirectory,dataFile,filenamePostfix),testMemPercentage,useHalfFloats)
dataBVH = readCSVFile("%s/%s_%s%s.csv" % (datasetDirectory,outputMode,dataFile,filenamePostfix),testMemPercentage,useHalfFloats)
groundTruthTest = dict()
groundTruthTest['labelIn'] = data2D['label']
groundTruthTest['in'] = data2D['body']
groundTruthTest['labelOut'] = dataBVH['label']
groundTruthTest['out'] = dataBVH['body']
checkIfTestAndTrainListsAreOk(groundTruthTest,groundTruthTrain)
#---------------------------------------------------------------
RAMAfter = getRAMInformation()
print("After reading the dataset | free RAM ",RAMAfter['free']," KB");
#----------------------------------------------------------------------------------------------------------
endAt = time.time()
print("Memory Occupied ",(RAMAfter['used']-RAMBefore['used'])/1024," MB")
print("Time required to load from disk was ",(endAt-startAt)/60," mins")
#
print("\n-------------------------------------------------");
#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------
#We have lodaded everything as normal but we only want to do identification in this run!
if (doOnlyIdentification):
identifyProblematicOutput(mapPath)
sys.exit(0)
#At this point we are ready..
#We should free whatever memory we have allocated because it is training time..!
#------------------------
gc.collect()
#------------------------
#Select a MocapNET class from tensorflow/tensorrt/onnx/tf-lite engines
from MocapNET import easyMocapNETConstructor
mnet = easyMocapNETConstructor(engine,
doProfiling=doProfiling,
doHCDPostProcessing = doHCDPostProcessing,
hcdLearningRate = hcdLearningRate,
hcdEpochs = hcdEpochs,
hcdIterations = hcdIterations
) #<- do not use post processing to actually evaluate MocapNET NN
mnet.test()
from align3DPoints import compareGroundTruthToPrediction
# Use body to grab all joints
#python3 evaluateMocapNET.py --mem 50 --config dataset/body_configuration.json --all upperbody
#-----------------------------------------------
totalSamples = 0
totalError = 0.0
averageErrorDistances = list()
allErrorDistanceSamples = list()
errorSamplesPerJoint = dict()
#-----------------------------------------------
numberOfSamples = len(groundTruthTest['in'])
#Declare joints we are interested in..
#-----------------------------------------
JOINTS_TO_COMPARE=list()
if (compareAllJoints==0):
JOINTS_TO_COMPARE.append('neck') #0
JOINTS_TO_COMPARE.append('rshoulder') #1
JOINTS_TO_COMPARE.append('relbow') #2
JOINTS_TO_COMPARE.append('rhand') #3
JOINTS_TO_COMPARE.append('lshoulder') #4
JOINTS_TO_COMPARE.append('lelbow') #5
JOINTS_TO_COMPARE.append('lhand') #6
JOINTS_TO_COMPARE.append('hip') #7
JOINTS_TO_COMPARE.append('rhip') #8
JOINTS_TO_COMPARE.append('rknee') #9
JOINTS_TO_COMPARE.append('rfoot') #10
JOINTS_TO_COMPARE.append('lhip') #11
JOINTS_TO_COMPARE.append('lknee') #12
JOINTS_TO_COMPARE.append('lfoot') #13
else:
print(bcolors.FAIL,"TODO: IMPLEMENT COMPARE ALL JOINTS",bcolors.ENDC)
sys.exit(1)
#Allocate errors per joint
#-----------------------------------------
if (len(JOINTS_TO_COMPARE)==0):
numberOfJoints = len(configuration["hierarchy"])
for jID in range (0,numberOfJoints):
jointName = configuration["hierarchy"][jID]["joint"].lower()
errorSamplesPerJoint[jointName]=list()
JOINTS_TO_COMPARE.append(jointName)
else:
numberOfJoints = len(JOINTS_TO_COMPARE)
for jID in range (0,numberOfJoints):
jointName = JOINTS_TO_COMPARE[jID].lower()
errorSamplesPerJoint[jointName]=list()
#-----------------------------------------
print("Doing evaluation based on errors of the following ",numberOfJoints," joints : ",JOINTS_TO_COMPARE)
f = open("evaluateMocapNET.lastresults.csv", "w")
#Append CSV Log
f.write("mae_from_%u_to_%u_step_%u\n" % (0,numberOfSamples,step))
framerates = list()
beginTime = time.time()
evaluationMode = "?"
frameNumber = 0 #<- simpler counter for nn subsampling
for thisSample in range(0,numberOfSamples,step) :
thisInputRaw = groundTruthTest['in'][thisSample]
correctOutputRaw = groundTruthTest['out'][thisSample]
regressedResult = dict()
correctResult = dict()
#DATA Looks like this : { 'labelInNoNSDM':names2D, 'labelIn':inputLabels, 'in':npInput, 'labelOut':outputLabels, 'out':npOutput };
thisInput = dict()
for i in range(0,len(groundTruthTest['labelIn'])):
thisLabelIn = groundTruthTest['labelIn'][i].lower()
if ( ( thisLabelIn.find("edm-")== -1 ) and ( thisLabelIn.find("nsrm-")== -1 ) ):
thisInput[thisLabelIn] = float(groundTruthTest['in'][thisSample][i])
#If we are dealing with just BVH output use training BVH output to extract 3D data
#------------------------------------------------------------------------------------------------------
if outputMode=="bvh":# (not 'outputMode' in configuration) or (configuration['outputMode'] == "bvh"):
evaluationMode="bvh"
correctOutput = dict()
for i in range(0,len(groundTruthTest['labelOut'])):
correctOutput[groundTruthTest['labelOut'][i]] = float(groundTruthTest['out'][thisSample][i])
correctResult = ground3DJointsFromBVH(mnet.bvh,correctOutput)
#------------------------------------------------------------------------------------------------------
#If we are dealing with 3D output immediately use training 3D output to compare 3D data
elif outputMode=="3d":# ('outputMode' in configuration) and (configuration['outputMode'] == "3d"):
evaluationMode="3d"
correctResult = dict()
for i in range(0,len(groundTruthTest['labelOut'])):
correctResult[groundTruthTest['labelOut'][i]] = float(groundTruthTest['out'][thisSample][i])
#------------------------------------------------------------------------------------------------------
startTime = time.time()
#================================================
#================================================
doNN = 1
if (doHCDPostProcessing) and (doNNEveryNFrames>1):
doNN = (frameNumber%doNNEveryNFrames)==0
#print("Frame ",frameNumber," doNN = ",doNN)
regressedResult = mnet.predict3DJoints(thisInput,runNN=doNN)
#================================================
#================================================
endTime = time.time()
processing_time = endTime - startTime
framerates.append(float(processing_time))
if (processing_time==0):
processing_time=1
#------------------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------------------
# Debug visualization (we shouldn't be doing a visualization in evaluateMocapNET.py)
#------------------------------------------------------------------------------------------------------
if (doDebugVisualization):
import cv2
import numpy as np
bgcolor = 30
annotated_image = np.full((1080,1920,3), bgcolor , dtype=np.uint8) #<--
from MocapNETVisualization import visualizeMocapNETEnsemble
image,plotImage = visualizeMocapNETEnsemble(mnet,annotated_image)
cv2.imshow('MocapNET 4 Evaluation', image)
if cv2.waitKey(1) & 0xFF == 27:
break
#------------------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------------------
#------------------------------------------------------------------------------------------------------
#print("\n\n\n\nthisInput ",thisInput)
#print("\n\n\n\ncorrectOutput ",correctOutput)
#print("\n\n\n\regressedResult ",regressedResult)
#print("\n\n\n\n")
stats = compareGroundTruthToPrediction(
configuration,
correctResult, #<- We want to compare this
regressedResult, #<- to This
doProcrustes=doProcrustes,
allowProcrustesToChangeScale=1,
jointsToCompare=JOINTS_TO_COMPARE
)
#Append CSV log
f.write("%f\n" % (float(stats["meanAverageError"])))
#print("\nstats ",stats)
print("\nstats={",end="")
for k in stats.keys():
print("'%s':%0.2f, "%(k,stats[k]),end="")
print("}\n")
progressMessage = "Progress : %0.2f %% %u / %u %0.2f fps" % ((100*thisSample)/numberOfSamples,thisSample,numberOfSamples,1/processing_time)
print(progressMessage," Evaluation Mode :",bcolors.OKGREEN,evaluationMode,bcolors.ENDC,"\n\n\n\n")
#numberOfJoints = len(configuration["hierarchy"])
for jID in range (0,numberOfJoints):
jointName = JOINTS_TO_COMPARE[jID]
if jointName in stats:
allErrorDistanceSamples.append(float(stats[jointName]))
errorSamplesPerJoint[jointName].append(float(stats[jointName]))
totalSamples = totalSamples + 1
totalError = totalError + float(stats[jointName])
else:
print("Could not find ",jointName)
averageErrorDistances.append(float(stats["meanAverageError"]))
frameNumber = frameNumber + 1 #<- simpler counter for nn subsampling
endTime = time.time()
print("Finished in ",(endTime-beginTime)/60," minutes")
f.close()
import numpy as np
appendRAWResultsForGNUplot("alldistance.raw",averageErrorDistances)
#------------------------------------------
for jID in range (0,numberOfJoints):
jointName = JOINTS_TO_COMPARE[jID]
appendRAWResultsForGNUplot("joint_%s.raw"%jointName,errorSamplesPerJoint[jointName])
#------------------------------------------
meanProcessingTime = np.mean(framerates)
minimum = np.min(averageErrorDistances)
maximum = np.max(averageErrorDistances)
median = np.median(averageErrorDistances)
mean = np.mean(averageErrorDistances)
std = np.std(averageErrorDistances)
var = np.var(averageErrorDistances)
#------------------------------------------
title_string=" Min/Max=%0.1f/%0.1f,Mean=%0.1f,Median=%0.1f,Std=%0.1f,Var=%0.1f" % (minimum,maximum,mean,median,std,var)
#Show a summary
#------------------------------------------
print("Flops : ",mnet.getModelFlops()," Model Params : ",mnet.getModelParameters())
print("Upperbody Decomposition : ",mnet.ensemble["upperbody"].configuration["decompositionType"]," | Input Size : ",mnet.ensemble["upperbody"].configuration["PCADimensionsKept"])
print("EDM upperbody : ",mnet.ensemble["upperbody"].configuration["EDM"])
print("eNSRM upperbody : ",mnet.ensemble["upperbody"].configuration["eNSRM"])
print("Lowerbody Decomposition : ",mnet.ensemble["lowerbody"].configuration["decompositionType"]," | Input Size : ",mnet.ensemble["lowerbody"].configuration["PCADimensionsKept"])
print("EDM lowerbody : ",mnet.ensemble["lowerbody"].configuration["EDM"])
print("eNSRM lowerbody : ",mnet.ensemble["lowerbody"].configuration["eNSRM"])
print("Sample Step : %u / Total Samples %u "%(step,numberOfSamples))
print("Procrustes : %u"%doProcrustes)
print("Mean Framerate : %0.2f fps"%(1/meanProcessingTime))
print("doHCDPostProcessing = ",doHCDPostProcessing)
print("doNNEveryNFrames = ",doNNEveryNFrames)
print("hcdLearningRate = ",hcdLearningRate)
print("hcdEpochs = ",hcdEpochs)
print("hcdIterations = ",hcdIterations)
print("Result : %s "%title_string)
notification("MocapNET Network Evaluation","Done evaluating -> %s " % (title_string))
f = open(label+"_results.csv", "w")
f.write("Params,PCA,FPS,Min,Max,Median,Mean,Std,Var,HCD,NNSubsample,HCD_LR,HCD_Epochs,HCD_Iterations\n")
f.write("%0.2f"%(mnet.getModelParameters()/1000.0))
f.write(",")
f.write(str(mnet.ensemble["upperbody"].configuration["PCADimensionsKept"]))
f.write(",")
f.write("%0.2f"%(1/meanProcessingTime))
f.write(",")
f.write(str(minimum))
f.write(",")
f.write(str(maximum))
f.write(",")
f.write(str(median))
f.write(",")
f.write(str(mean))
f.write(",")
f.write(str(std))
f.write(",")
f.write(str(var))
f.write(",")
f.write(str(doHCDPostProcessing))
f.write(",")
f.write(str(doNNEveryNFrames))
f.write(",")
f.write(str(hcdLearningRate))
f.write(",")
f.write(str(hcdEpochs))
f.write(",")
f.write(str(hcdIterations))
f.write("\n")
f.close()
#Plotting !
#------------------------------------------
import matplotlib.pyplot as plt
fig, ax = plt.subplots(figsize=(8, 4))
#------------------------------------------
x = averageErrorDistances
mu = 200
sigma = 25
n_bins = 100
# plot the cumulative histogram
n, bins, patches = ax.hist(x, n_bins, density=True, histtype='step',cumulative=True, label='Accuracy')
#------------------------------------------
# Add a line showing the expected distribution.
y = ((1 / (np.sqrt(2 * np.pi) * sigma)) *
np.exp(-0.5 * (1 / sigma * (bins - mu))**2))
y = y.cumsum()
y /= y[-1]
#------------------------------------------
#ax.plot(bins, y, 'k--', linewidth=1.5, label='Theoretical')
# tidy up the figure
ax.grid(True)
ax.legend(loc='right')
ax.set_title('Cumulative step histograms')
ax.set_xlabel('3D Error (mm)')
ax.set_ylabel('Likelihood of occurrence')
#------------------------------------------
#plt.show()
print("Done")
if __name__ == '__main__':
#When working standalone assume simple csv backend
emptyData = dict()
emptyTestData = { 'labelInNoNSDM': [], 'labelIn': [], 'in': [], 'labelOut':[], 'out': [] }
useCSVBackend=1
evaluateMocapNET(useCSVBackend,emptyData,emptyTestData)
Reference in New Issue View Git Blame Copy Permalink