mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-16 00:13:07 +00:00
Justin John
353 lines
13 KiB
Python
353 lines
13 KiB
Python
import matplotlib as mp
|
|
import matplotlib.pyplot as plt
|
|
import numpy as np
|
|
import os
|
|
import sys
|
|
import time
|
|
import array
|
|
import csv
|
|
|
|
# Fixing random state for reproducibility
|
|
np.random.seed(19680801)
|
|
|
|
|
|
|
|
class bcolors:
|
|
HEADER = '\033[95m'
|
|
OKBLUE = '\033[94m'
|
|
OKGREEN = '\033[92m'
|
|
WARNING = '\033[93m'
|
|
FAIL = '\033[91m'
|
|
ENDC = '\033[0m'
|
|
BOLD = '\033[1m'
|
|
UNDERLINE = '\033[4m'
|
|
|
|
"""
|
|
Bytes to KB/MB/GB/TB converter
|
|
"""
|
|
def convert_bytes(num):
|
|
"""
|
|
this function will convert bytes to MB.... GB... etc
|
|
"""
|
|
step_unit = 1000.0 #1024 bad the size
|
|
|
|
for x in ['bytes', 'KB', 'MB', 'GB', 'TB']:
|
|
if num < step_unit:
|
|
return "%3.1f %s" % (num, x)
|
|
num /= step_unit
|
|
|
|
"""
|
|
Check if a file exists
|
|
"""
|
|
def checkIfFileExists(filename):
|
|
return os.path.isfile(filename)
|
|
|
|
|
|
"""
|
|
Count the number of lines by parsing the file inside python
|
|
"""
|
|
def getNumberOfLines(filename):
|
|
#import socket
|
|
#print("Hostname is ",socket.gethostname())
|
|
#if (socket.gethostname()=="ammar-kriti"):
|
|
# print(bcolors.FAIL,"RETURNING FIXED NUMBER OF LINES FOR SPEED IN MY SLOW PC",bcolors.ENDC)
|
|
# return 3858095
|
|
print("Counting number of lines in file ",filename)
|
|
with open(filename) as f:
|
|
return sum(1 for line in f)
|
|
|
|
|
|
"""
|
|
typically we work with data[NUMBER_OF_SAMPLES,NUMBER_OF_COLUMNS]
|
|
calling splitNumpyArray(data,4,4,0) means get back columns 4,5,6,7 so it will return a data[NUMBER_OF_SAMPLES,4]
|
|
"""
|
|
def splitNumpyArray(data,column,columnsToTake,useHalfFloats=0):
|
|
#-------------------------------------
|
|
dtypeSelected=np.dtype(np.float32)
|
|
if (useHalfFloats):
|
|
dtypeSelected=np.dtype(np.float16)
|
|
#-------------------------------------
|
|
numberOfSamples=len(data)
|
|
npOutput = np.full([numberOfSamples,columnsToTake],fill_value=0,dtype=dtypeSelected,order='C')
|
|
|
|
for outCol in range(0,columnsToTake):
|
|
for num in range(0,numberOfSamples):
|
|
npOutput[num,outCol]=float(data[num,column+outCol])
|
|
#-------------------------------------
|
|
return npOutput;
|
|
|
|
#---------------------------------------------------------------------------------------------------------------------------------------------
|
|
#---------------------------------------------------------------------------------------------------------------------------------------------
|
|
#---------------------------------------------------------------------------------------------------------------------------------------------
|
|
def readCSVFile(filename,memPercentage,useHalfFloats=0):
|
|
print("CSV file :",filename,"..\n")
|
|
|
|
if (not checkIfFileExists(filename)):
|
|
print( bcolors.FAIL + "Input file "+filename+" does not exist, cannot read ground truth.." + bcolors.ENDC)
|
|
print("Current Directory was "+os.getcwd())
|
|
sys.exit(1)
|
|
start = time.time()
|
|
|
|
dtypeSelected=np.dtype(np.float32)
|
|
dtypeSelectedByteSize=int(dtypeSelected.itemsize)
|
|
if (useHalfFloats):
|
|
dtypeSelected=np.dtype(np.float16)
|
|
dtypeSelectedByteSize=int(dtypeSelected.itemsize)
|
|
|
|
progress=0.0
|
|
#useSquared=1
|
|
sampleNumber=0
|
|
receivedHeader=0
|
|
inputNumberOfColumns=0
|
|
outputNumberOfColumns=0
|
|
|
|
inputLabels=list()
|
|
|
|
#-------------------------------------------------------------------------------------------------
|
|
numberOfSamplesInput=getNumberOfLines(filename)-2
|
|
print(" Input file has ",numberOfSamplesInput," training samples\n")
|
|
#-------------------------------------------------------------------------------------------------
|
|
|
|
|
|
numberOfSamples = numberOfSamplesInput
|
|
numberOfSamplesLimit=int(numberOfSamples*memPercentage)
|
|
#-------------------------------------------------------------------------------------------------
|
|
if (memPercentage==0.0):
|
|
print("readGroundTruthFile was asked to occupy 0 memory so this probably means we just want one record")
|
|
numberOfSamplesLimit=2
|
|
if (memPercentage>1.0):
|
|
print("Memory Limit will be interpreted as a raw value..")
|
|
numberOfSamplesLimit=int(memPercentage)
|
|
#-------------------------------------------------------------------------------------------------
|
|
|
|
|
|
thisInput = array.array('f')
|
|
#---------------------------------
|
|
|
|
fi = open(filename, "r")
|
|
readerIn = csv.reader( fi , delimiter =',', skipinitialspace=True)
|
|
for rowIn in readerIn:
|
|
#------------------------------------------------------
|
|
if (not receivedHeader): #use header to get labels
|
|
#------------------------------------------------------
|
|
inputNumberOfColumns=len(rowIn)
|
|
inputLabels = list(rowIn[i] for i in range(0,inputNumberOfColumns) )
|
|
print("Number of Input elements : ",len(inputLabels))
|
|
#------------------------------------------------------
|
|
|
|
if (memPercentage==0):
|
|
print("Will only return labels\n")
|
|
return {'labels':inputLabels};
|
|
#i=0
|
|
#print("class Input(Enum):")
|
|
#for label in inputLabels:
|
|
# print(" ",label," = ",i," #",int(i/3))
|
|
# print(" ",label,"=",int(i/3))
|
|
# i=i+1
|
|
|
|
#---------------------------------
|
|
# Allocate Lists
|
|
#---------------------------------
|
|
for i in range(inputNumberOfColumns):
|
|
thisInput.append(0.0)
|
|
#---------------------------------
|
|
|
|
|
|
#---------------------------------
|
|
# Allocate Numpy Arrays
|
|
#---------------------------------
|
|
inputSize=0
|
|
startCompressed=0
|
|
inputSize=inputNumberOfColumns
|
|
startCompressed=inputNumberOfColumns
|
|
|
|
npInputBytesize=0+numberOfSamplesLimit * inputSize * dtypeSelectedByteSize
|
|
print(" Input file on disk has a shape of [",numberOfSamples,",",inputSize,"]")
|
|
print(" Input we will read has a shape of [",numberOfSamplesLimit,",",inputSize,"]")
|
|
print(" Input will occupy ",convert_bytes(npInputBytesize)," of RAM\n")
|
|
npInput = np.full([numberOfSamplesLimit,inputSize],fill_value=0,dtype=dtypeSelected,order='C')
|
|
#----------------------------------------------------------------------------------------------------------
|
|
receivedHeader=1
|
|
#sys.exit(0)
|
|
else:
|
|
#-------------------------------------------
|
|
# First convert our string INPUT to floats
|
|
#-------------------------------------------
|
|
for i in range(inputNumberOfColumns):
|
|
try:
|
|
thisInput[i]=float(rowIn[i])
|
|
except ValueError:
|
|
thisInput[i]=0.0
|
|
#-------------------------------------------
|
|
for num in range(0,inputNumberOfColumns):
|
|
npInput[sampleNumber,num]=float(thisInput[num]);
|
|
#-------------------------------------------
|
|
sampleNumber=sampleNumber+1
|
|
|
|
if (numberOfSamples>0):
|
|
progress=sampleNumber/numberOfSamplesLimit
|
|
|
|
if (sampleNumber%1000==0) :
|
|
progressString = "%0.2f"%float(100*progress)
|
|
print("\rReading from disk (",sampleNumber,") - ",progressString," % \r", end="", flush=True)
|
|
|
|
if (numberOfSamplesLimit<=sampleNumber):
|
|
print("\rStopping reading file to obey memory limit given by parameter --mem ",memPercentage,"\n")
|
|
break
|
|
#-------------------------------------------
|
|
fi.close()
|
|
del readerIn
|
|
|
|
|
|
print("\n read, Samples: ",sampleNumber,", was expecting ",numberOfSamples," samples\n")
|
|
print(npInput.shape)
|
|
|
|
totalNumberOfBytes=npInput.nbytes;
|
|
totalNumberOfGigaBytes=totalNumberOfBytes/1073741824;
|
|
print("GPU Size Occupied by data = ",totalNumberOfGigaBytes," GB \n")
|
|
|
|
end = time.time()
|
|
print("Time elapsed : ",(end-start)/60," mins")
|
|
#---------------------------------------------------------------------
|
|
return {'label':inputLabels, 'body':npInput };
|
|
|
|
|
|
def resolve(theList,theName):
|
|
result = 0
|
|
for item in theList:
|
|
if (item==theName):
|
|
return result,True
|
|
result = result + 1
|
|
return result,False
|
|
|
|
if __name__== "__main__":
|
|
|
|
filename="report.csv"
|
|
output="out.png"
|
|
jointName=" HCD FineTune "
|
|
#----------------------------------------------
|
|
if (len(sys.argv)>1):
|
|
print('Argument List:', str(sys.argv))
|
|
for i in range(0, len(sys.argv)):
|
|
if (sys.argv[i]=="--from"):
|
|
filename=sys.argv[i+1]
|
|
data = readCSVFile(filename,1.0)
|
|
#----------------------------------------------
|
|
if (filename=="report.csv"):
|
|
data = readCSVFile(filename,1.0)
|
|
#----------------------------------------------
|
|
|
|
xCol=1
|
|
yCol=6
|
|
zCol=8
|
|
target1Col=11
|
|
target2Col=12
|
|
|
|
xs = None
|
|
ys = None
|
|
zs = None
|
|
vs = None
|
|
v2s = None
|
|
|
|
#https://matplotlib.org/stable/gallery/color/colormap_reference.html
|
|
colorMapString = "Greens_r"
|
|
#supported values are 'Accent', 'Accent_r', 'Blues', 'Blues_r', 'BrBG', 'BrBG_r', 'BuGn', 'BuGn_r', 'BuPu', 'BuPu_r', 'CMRmap', 'CMRmap_r', 'Dark2', 'Dark2_r', 'GnBu', 'GnBu_r', 'Greens', 'Greens_r', 'Greys', 'Greys_r', 'OrRd', 'OrRd_r', 'Oranges', 'Oranges_r', 'PRGn', 'PRGn_r', 'Paired', 'Paired_r', 'Pastel1', 'Pastel1_r', 'Pastel2', 'Pastel2_r', 'PiYG', 'PiYG_r', 'PuBu', 'PuBuGn', 'PuBuGn_r', 'PuBu_r', 'PuOr', 'PuOr_r', 'PuRd', 'PuRd_r', 'Purples', 'Purples_r', 'RdBu', 'RdBu_r', 'RdGy', 'RdGy_r', 'RdPu', 'RdPu_r', 'RdYlBu', 'RdYlBu_r', 'RdYlGn', 'RdYlGn_r', 'Reds', 'Reds_r', 'Set1', 'Set1_r', 'Set2', 'Set2_r', 'Set3', 'Set3_r', 'Spectral', 'Spectral_r', 'Wistia', 'Wistia_r', 'YlGn', 'YlGnBu', 'YlGnBu_r', 'YlGn_r', 'YlOrBr', 'YlOrBr_r', 'YlOrRd', 'YlOrRd_r', 'afmhot', 'afmhot_r', 'autumn', 'autumn_r', 'binary', 'binary_r', 'bone', 'bone_r', 'brg', 'brg_r', 'bwr', 'bwr_r', 'cividis', 'cividis_r', 'cool', 'cool_r', 'coolwarm', 'coolwarm_r', 'copper', 'copper_r', 'cubehelix', 'cubehelix_r', 'flag', 'flag_r', 'gist_earth', 'gist_earth_r', 'gist_gray', 'gist_gray_r', 'gist_heat', 'gist_heat_r', 'gist_ncar', 'gist_ncar_r', 'gist_rainbow', 'gist_rainbow_r', 'gist_stern', 'gist_stern_r', 'gist_yarg', 'gist_yarg_r', 'gnuplot', 'gnuplot2', 'gnuplot2_r', 'gnuplot_r', 'gray', 'gray_r', 'hot', 'hot_r', 'hsv', 'hsv_r', 'inferno', 'inferno_r', 'jet', 'jet_r', 'magma', 'magma_r', 'nipy_spectral', 'nipy_spectral_r', 'ocean', 'ocean_r', 'pink', 'pink_r', 'plasma', 'plasma_r', 'prism', 'prism_r', 'rainbow', 'rainbow_r', 'seismic', 'seismic_r', 'spring', 'spring_r', 'summer', 'summer_r', 'tab10', 'tab10_r', 'tab20', 'tab20_r', 'tab20b', 'tab20b_r', 'tab20c', 'tab20c_r', 'terrain', 'terrain_r', 'turbo', 'turbo_r', 'twilight', 'twilight_r', 'twilight_shifted', 'twilight_shifted_r', 'viridis', 'viridis_r', 'winter', 'winter_r'
|
|
|
|
|
|
xOk = False
|
|
yOk = False
|
|
zOk = False
|
|
target1Ok = False
|
|
target2Ok = False
|
|
|
|
if (len(sys.argv)>1):
|
|
print('Argument List:', str(sys.argv))
|
|
for i in range(0, len(sys.argv)):
|
|
if (sys.argv[i]=="--colormap"):
|
|
colorMapString=sys.argv[i+1]
|
|
if (sys.argv[i]=="--to"):
|
|
output=sys.argv[i+1]
|
|
if (sys.argv[i]=="--title"):
|
|
jointName=sys.argv[i+1]
|
|
if (sys.argv[i]=="--label"):
|
|
jointName=sys.argv[i+1]
|
|
if (sys.argv[i]=="-x"):
|
|
xCol,xOk = resolve(data["label"],sys.argv[i+1])
|
|
if (sys.argv[i]=="-y"):
|
|
yCol,yOk = resolve(data["label"],sys.argv[i+1])
|
|
if (sys.argv[i]=="-z"):
|
|
zCol,zOk = resolve(data["label"],sys.argv[i+1])
|
|
if (sys.argv[i]=="-v2"):
|
|
target2Col,target2Ok = resolve(data["label"],sys.argv[i+1])
|
|
if (sys.argv[i]=="-v1"):
|
|
target1Col,target1Ok = resolve(data["label"],sys.argv[i+1])
|
|
|
|
|
|
|
|
#-------------------------
|
|
xLabel=data["label"][xCol]
|
|
yLabel=data["label"][yCol]
|
|
zLabel=data["label"][zCol]
|
|
#-------------------------
|
|
viewAzimuth=45
|
|
viewElevation=45
|
|
transparency=0.6
|
|
scale = 0.12 #Smaller is smaller :P
|
|
|
|
|
|
fig = plt.figure()
|
|
ax = fig.add_subplot(projection='3d')
|
|
|
|
if xOk:
|
|
xs = splitNumpyArray(data["body"],xCol,1) # 1 = learning rate
|
|
else:
|
|
print("Error Resolving X")
|
|
|
|
if yOk:
|
|
ys = splitNumpyArray(data["body"],yCol,1) # 6 = iteration
|
|
else:
|
|
print("Error Resolving Y")
|
|
|
|
if zOk:
|
|
zs = splitNumpyArray(data["body"],zCol,1) # 8 = langevin
|
|
else:
|
|
print("Error Resolving Z")
|
|
|
|
if target1Ok:
|
|
vs = splitNumpyArray(data["body"],target1Col,1) # 10 = mae
|
|
else:
|
|
print("Error Resolving Color (v1)")
|
|
|
|
if target2Ok:
|
|
v2s = splitNumpyArray(data["body"],target2Col,1) # 10 = mae
|
|
v2sMean = np.mean(v2s)
|
|
v2s = v2s * (100.0/v2sMean)
|
|
else:
|
|
print("Error Resolving Size (v2)")
|
|
|
|
ax.view_init(viewAzimuth,viewElevation)
|
|
|
|
#cm = plt.cm.get_cmap('RdYlBu')
|
|
#cm = plt.cm.get_cmap('hot')
|
|
cm = mp.colormaps.get_cmap(colorMapString)
|
|
|
|
|
|
if target1Ok and target2Ok:
|
|
sc = ax.scatter(xs, ys, zs, c=vs, s=v2s , cmap=cm, alpha=transparency)
|
|
else:
|
|
sc = ax.scatter(xs, ys, zs, alpha=transparency)
|
|
|
|
|
|
|
|
plt.colorbar(sc)
|
|
|
|
ax.set_title(jointName) # Title of the plot
|
|
ax.set_xlabel(xLabel)
|
|
ax.set_ylabel(yLabel)
|
|
ax.set_zlabel(zLabel)
|
|
|
|
fig.savefig(output)
|
|
plt.show()
|
|
|
|
|