Added new UI for the Temporal Sync Feature, made the software more robust including:

-Settings window can only be opened once
    -When the client can't open a device, it tries multiple times
    -Recording bins are now saved with a unique ID per device, to allow multiple clients to record on the same PC
    -Synchronized & Unsynchronized devices can be both used in one server
    -Included a workaround for a bug in the Azure Kinect SDK regarding exposure (microsoft/Azure-Kinect-Sensor-SDK#1378)

src/LiveScanClient/azureKinectCapture.cpp

@@ -24,20 +24,51 @@ AzureKinectCapture::~AzureKinectCapture()
 /// <param name="asSubordinate">Initalizes this devices as subordinate</param>
 /// <param name="syncOffsetMultiplier">Should only be set when initializing as Subordinate. Each subordinate should have a unique, ascending value</param>
 /// <returns></returns>
-bool AzureKinectCapture::Initialize(bool asMaster, bool asSubordinate, int syncOffsetMultiplier)
+bool AzureKinectCapture::Initialize(SYNC_STATE state, int syncOffsetMultiplier)
 {
 	uint32_t count = k4a_device_get_installed_count();
 	int deviceIdx = 0;
 
+	//We save the deviceId of this Client.
+	//When the cameras are reinitialized during runtime, we can then gurantee
+	//that each LiveScan instance uses the same device as before (In case two or more Kinects are connected to the same PC)
+	//A device ID of -1 means that no Kinects has been successfully initalized yet (only happens when the Client starts)
+	if (deviceIDForRestart != -1) {
+		deviceIdx = deviceIDForRestart;
+	}
+
 	kinectSensor = NULL;
 	while (K4A_FAILED(k4a_device_open(deviceIdx, &kinectSensor)))
 	{
-		deviceIdx++;
-		if (deviceIdx >= count)
+		
+		if (deviceIDForRestart == -1) 
 		{
-			bInitialized = false;
-			return bInitialized;
+			deviceIdx++;
+			if (deviceIdx >= count)
+			{
+				bInitialized = false;
+				return bInitialized;
+			}
 		}
+
+		
+		else {
+
+			//Sometimes the cameras fail to reinitialize, so we try to initialize them three times with a slight delay before failing
+			if (restartAttempts > 2) 
+			{
+				bInitialized = false;
+				return bInitialized;
+			}
+
+			else {
+
+				restartAttempts++;
+				Sleep(200);
+			}
+			
+		}
+		
 	}
 
 	k4a_device_configuration_t config = K4A_DEVICE_CONFIG_INIT_DISABLE_ALL;
@@ -47,12 +78,12 @@ bool AzureKinectCapture::Initialize(bool asMaster, bool asSubordinate, int syncO
 	config.depth_mode = K4A_DEPTH_MODE_NFOV_UNBINNED;
 	config.synchronized_images_only = true;
 
-	if (asMaster) 
+	if (state == Master) 
 	{
 		config.wired_sync_mode = K4A_WIRED_SYNC_MODE_MASTER;
 	}
 
-	else if (asSubordinate) 
+	else if (state == Subordinate) 
 	{
 		config.wired_sync_mode = K4A_WIRED_SYNC_MODE_SUBORDINATE;
 		//Sets the offset on subordinate devices. Should be a multiple of 160, each subordinate having a different multiplier in ascending order.
@@ -75,10 +106,24 @@ bool AzureKinectCapture::Initialize(bool asMaster, bool asSubordinate, int syncO
 		return bInitialized;
 	}
 
+	//Workaround for a bug. When the camera starts in manual Exposure mode, the brightness of the RBG image
+	//is much lower than in auto exposure mode. To prevent this, we first set the camera to auto exposure mode and 
+	//then switch to manual mode again if it has been enabled before
+
+	if (autoExposureEnabled == false) {
+
+		k4a_device_set_color_control(kinectSensor, K4A_COLOR_CONTROL_EXPOSURE_TIME_ABSOLUTE, K4A_COLOR_CONTROL_MODE_AUTO, 0);
+
+		//Give it a second to adjust
+		Sleep(1000);
+
+		SetExposureState(false, exposureTimeStep);
+	}
+
 	transformation = k4a_transformation_create(&calibration);
 
 	//If this device is a subordinate, it is expected to start capturing at a later time (When the master has started), so we skip this check 
-	if (!asSubordinate) 
+	if (state != Subordinate) 
 	{
 		std::chrono::time_point<std::chrono::system_clock> start = std::chrono::system_clock::now();
 		bool bTemp;
@@ -100,6 +145,9 @@ bool AzureKinectCapture::Initialize(bool asMaster, bool asSubordinate, int syncO
 	serialNumber = std::string(serialNoSize, '\0');
 	k4a_device_get_serialnum(kinectSensor, (char*)serialNumber.c_str(), &serialNoSize);
 
+	deviceIDForRestart = deviceIdx;
+	restartAttempts = 0;
+
 	return bInitialized;
 }
 
@@ -173,6 +221,41 @@ bool AzureKinectCapture::AcquireFrame()
 	return true;
 }
 
+/// <summary>
+/// Enables/Disables Auto Exposure and/or sets the exposure to a step value between -11 and -5 
+/// The kinect supports exposure values up to 1, but these are only available in lower FPS modes (5 or 15 FPS)
+/// For further information on how this value translates to ms-Values, please take a look
+/// at this table: https://github.com/microsoft/Azure-Kinect-Sensor-SDK/blob/develop/src/color/color_priv.h
+/// </summary>
+/// <param name="exposureStep">The Exposure Step between -11 and 1</param>
+void AzureKinectCapture::SetExposureState(bool enableAutoExposure, int exposureStep)
+{
+	if (bInitialized) 
+	{
+		if (enableAutoExposure)
+		{
+			k4a_device_set_color_control(kinectSensor, K4A_COLOR_CONTROL_EXPOSURE_TIME_ABSOLUTE, K4A_COLOR_CONTROL_MODE_AUTO, 0);
+
+			autoExposureEnabled = true;
+		}
+
+		else
+		{
+			//Formula copied from here: https://github.com/microsoft/Azure-Kinect-Sensor-SDK/blob/7cd8683a1a71b8baebef4a3537e6edd8639d1e95/examples/k4arecorder/main.cpp#L333
+			float absExposure = (exp2f((float)exposureStep) * 1000000.0f);
+			//We add 0.5 because C++ always truncates when converting to an integer. 
+			//This ensures that values will always be rounded correctly
+			float absExposureRoundingMargin = absExposure + 0.5;
+			int32_t absoluteExposureInt = (int32_t)absExposureRoundingMargin;
+
+			k4a_device_set_color_control(kinectSensor, K4A_COLOR_CONTROL_EXPOSURE_TIME_ABSOLUTE, K4A_COLOR_CONTROL_MODE_MANUAL, absoluteExposureInt);
+
+			autoExposureEnabled = false;
+			exposureTimeStep = exposureStep;
+		}
+	}	
+}
+
 void AzureKinectCapture::UpdateDepthPointCloud()
 {
 	if (depthPointCloudImage == NULL)
@@ -348,3 +431,8 @@ uint64_t AzureKinectCapture::GetTimeStamp()
 	return currentTimeStamp;
 }
 
+int AzureKinectCapture::GetDeviceIndex() 
+{
+	return deviceIDForRestart;
+}
+

src/LiveScanClient/frameFileWriterReader.cpp

@@ -39,14 +39,14 @@ void FrameFileWriterReader::openCurrentFileForReading()
 	m_bFileOpenedForWriting = false;
 }
 
-void FrameFileWriterReader::openNewFileForWriting()
+void FrameFileWriterReader::openNewFileForWriting(int deviceID)
 {
 	closeFileIfOpened();
 
 	char filename[1024];
 	time_t t = time(0);
 	struct tm * now = localtime(&t);
-	sprintf(filename, "recording_%04d_%02d_%02d_%02d_%02d.bin", now->tm_year + 1900, now->tm_mon + 1, now->tm_mday, now->tm_hour, now->tm_min, now->tm_sec);
+	sprintf(filename, "recording_%01d_%04d_%02d_%02d_%02d_%02d.bin", deviceID, now->tm_year + 1900, now->tm_mon + 1, now->tm_mday, now->tm_hour, now->tm_min, now->tm_sec);
 	m_sFilename = filename; 
 	m_pFileHandle = fopen(filename, "wb");
 
@@ -86,10 +86,10 @@ bool FrameFileWriterReader::readFrame(std::vector<Point3s> &outPoints, std::vect
 }
 
 
-void FrameFileWriterReader::writeFrame(std::vector<Point3s> points, std::vector<RGB> colors, uint64_t timestamp)
+void FrameFileWriterReader::writeFrame(std::vector<Point3s> points, std::vector<RGB> colors, uint64_t timestamp, int deviceID)
 {
 	if (!m_bFileOpenedForWriting)
-		openNewFileForWriting();
+		openNewFileForWriting(deviceID);
 
 	FILE *f = m_pFileHandle;
 

src/LiveScanClient/liveScanClient.cpp

@@ -38,13 +38,13 @@ int APIENTRY wWinMain(
 }
 
 LiveScanClient::LiveScanClient() :
-    m_hWnd(NULL),
-    m_nLastCounter(0),
-    m_nFramesSinceUpdate(0),
-    m_fFreq(0),
-    m_nNextStatusTime(0LL),
-    m_pD2DFactory(NULL),
-    m_pDrawColor(NULL),
+	m_hWnd(NULL),
+	m_nLastCounter(0),
+	m_nFramesSinceUpdate(0),
+	m_fFreq(0),
+	m_nNextStatusTime(0LL),
+	m_pD2DFactory(NULL),
+	m_pDrawColor(NULL),
 	m_pDepthRGBX(NULL),
 	m_pCameraSpaceCoordinates(NULL),
 	m_pColorInDepthSpace(NULL),
@@ -56,13 +56,17 @@ LiveScanClient::LiveScanClient() :
 	m_bConnected(false),
 	m_bConfirmCaptured(false),
 	m_bConfirmCalibrated(false),
+	m_bConfirmRestartAsMaster(false),
 	m_bShowDepth(false),
 	m_bSocketThread(true),
 	m_bFrameCompression(true),
 	m_iCompressionLevel(2),
 	m_pClientSocket(NULL),
 	m_nFilterNeighbors(10),
-	m_fFilterThreshold(0.01f)
+	m_fFilterThreshold(0.01f),
+	m_bRestartingCamera(false),
+	m_bAutoExposureEnabled(true), // Which state the Auto Exposure should be set to
+	m_nExposureStep(-5)
 {
 	pCapture = new AzureKinectCapture();
 
@@ -207,7 +211,7 @@ void LiveScanClient::UpdateFrame()
 		if (m_bCaptureFrame)
 		{
 			uint64_t timeStamp = pCapture->GetTimeStamp();
-			m_framesFileWriterReader.writeFrame(m_vLastFrameVertices, m_vLastFrameRGB, timeStamp);
+			m_framesFileWriterReader.writeFrame(m_vLastFrameVertices, m_vLastFrameRGB, timeStamp, pCapture->GetDeviceIndex());
 			m_bConfirmCaptured = true;
 			m_bCaptureFrame = false;
 		}
@@ -277,7 +281,7 @@ LRESULT CALLBACK LiveScanClient::DlgProc(HWND hWnd, UINT message, WPARAM wParam,
             D2D1CreateFactory(D2D1_FACTORY_TYPE_SINGLE_THREADED, &m_pD2DFactory);
 
             // Get and initialize the default Kinect sensor as standalone
-			bool res = pCapture->Initialize(false, false, 0);
+			bool res = pCapture->Initialize(Standalone, 0);
 			if (res)
 			{
 				calibration.LoadCalibration(pCapture->serialNumber);
@@ -286,6 +290,8 @@ LRESULT CALLBACK LiveScanClient::DlgProc(HWND hWnd, UINT message, WPARAM wParam,
 
 				m_pCameraSpaceCoordinates = new Point3f[pCapture->nDepthFrameWidth * pCapture->nDepthFrameHeight];
 				m_pColorInDepthSpace = new RGB[pCapture->nDepthFrameWidth * pCapture->nDepthFrameHeight];
+
+				pCapture->SetExposureState(true, 0);
 			}
 			else
 			{
@@ -455,30 +461,69 @@ void LiveScanClient::HandleSocket()
 			//Determine if this device is a subordinate, master, or standalone
 			int jackState = pCapture->GetSyncJackState();
 
+			bool res = false;
+
 			switch (jackState)
 			{
 			case -1:
 				currentTempSyncState = SUBORDINATE;
+
 				//Restart this device as Subordinate, with a unique syncOffset (send by the server)
-				pCapture->Close();
-				pCapture->Initialize(false, true, syncOffset);
+				m_bRestartingCamera = true;
+
+
+				res = pCapture->Close();
+				if (!res) {
+					SetStatusMessage(L"Subordinate device failed to close! Restart Application!", 10000, true);
+					return;
+				}
+
+				res = pCapture->Initialize(Subordinate, syncOffset);
+				if (!res) {
+					SetStatusMessage(L"Subordinate device failed to reinitialize! Restart Application!", 10000, true);
+					return;
+				}
 				//Confirm to the server, that we set this device as subordinate
 				m_bConfirmTempSyncState = true;
+				m_bRestartingCamera = false;
 				break;
 
 			case 0: 
 				currentTempSyncState = MASTER;
+
 				//Only Close this device, as it needs to wait for all subordinates to start, before starting itself
-				pCapture->Close();
+				m_bRestartingCamera = true;
+
+				res = pCapture->Close();
+				if (!res) {
+					SetStatusMessage(L"Master device failed to close! Restart Application!", 10000, true);
+					return;
+				}
+
 				m_bConfirmTempSyncState = true;
 				break;
 
 			case 1://Device is Standalone
 				currentTempSyncState = STANDALONE;
+
 				//Restart this device as Standalone
-				pCapture->Close();
-				pCapture->Initialize(false, false, 0);
+				m_bRestartingCamera = true;
+
+				res = pCapture->Close();
+				if (!res) {
+					SetStatusMessage(L"Capture device failed to close! Restart Application!", 10000, true);
+					return;
+				}
+
+				res = pCapture->Initialize(Standalone, 0);
+
+				if (!res) {
+					SetStatusMessage(L"Capture device failed to reinitialize! Restart Application!", 10000, true);
+					return;
+				}
+
 				m_bConfirmTempSyncState = true;
+				m_bRestartingCamera = false;
 				break;
 			default:
 				break;
@@ -488,16 +533,39 @@ void LiveScanClient::HandleSocket()
 		//Sets this device as Standalone
 		else if (received[i] == MSG_SET_TEMPSYNC_OFF) {
 			currentTempSyncState = STANDALONE;
-			pCapture->Close();
-			pCapture->Initialize(false, false, 0);
+			m_bRestartingCamera = true;
+
+			bool res;
+
+			res = pCapture->Close();
+			if (!res) {
+				SetStatusMessage(L"Capture device failed to close! Restart Application!", 10000, true);
+				return;
+			}
+
+			res = pCapture->Initialize(Standalone, 0);
+
+			if (!res) {
+				SetStatusMessage(L"Capture device failed to reinitialize! Restart Application!", 10000, true);
+				return;
+			}
+
 			m_bConfirmTempSyncState = true;
+			m_bRestartingCamera = false;
 		}
 
 		//Got confirmation from the server that all subs have started, and we can now start the master 
 		else if (received[i] == MSG_START_MASTER) {
 			if (currentTempSyncState == MASTER) 
 			{
-				pCapture->Initialize(true, false, 0);
+				bool res = pCapture->Initialize(Master, 0);
+				if (!res) {
+					SetStatusMessage(L"Master device failed to reinitialize! Restart Application!", 10000, true);
+					return;
+				}
+
+				m_bConfirmRestartAsMaster = true;
+				m_bRestartingCamera = false;
 			}
 		}
 
@@ -562,6 +630,14 @@ void LiveScanClient::HandleSocket()
 				m_bFrameCompression = true;
 			else
 				m_bFrameCompression = false;
+			
+			m_bAutoExposureEnabled = (received[i] != 0);
+			i++;
+
+			m_nExposureStep = *(int*)(received.c_str() + i);
+			i += sizeof(int);
+
+			pCapture->SetExposureState(m_bAutoExposureEnabled, m_nExposureStep);
 
 			//so that we do not lose the next character in the stream
 			i--;
@@ -615,6 +691,18 @@ void LiveScanClient::HandleSocket()
 		{
 			m_framesFileWriterReader.closeFileIfOpened();
 		}
+
+		else if (received[i] == MSG_REQUEST_SYNC_JACK_STATE) 
+		{
+			int size = 3;
+			char* buffer = new char[size];
+			buffer[0] = MSG_SYNC_JACK_STATE;
+
+			buffer[1] = pCapture->GetSyncJackState() + 1; //Gets the current Sync Jack State and adds + 1, as we can't send a negative value
+
+			m_pClientSocket->SendBytes(buffer, size);
+			m_bConfirmTempSyncState = false;
+		}
 	}
 
 	if (m_bConfirmCaptured)
@@ -642,6 +730,14 @@ void LiveScanClient::HandleSocket()
 		m_bConfirmTempSyncState = false;
 	}
 
+	//Send validation to the server that the Master camera has started recording again
+	if (m_bConfirmRestartAsMaster) 
+	{
+		byteToSend = MSG_CONFIRM_MASTER_RESTART;
+		m_pClientSocket->SendBytes(&byteToSend, 1);
+		m_bConfirmRestartAsMaster = false;
+	}
+
 
 	if (m_bConfirmCalibrated)
 	{