From 76bf5e9dba87aac98a9c2d7ac723ae77d89bf2ce Mon Sep 17 00:00:00 2001 From: Christopher Remde Date: Wed, 14 Oct 2020 23:21:18 +0200 Subject: [PATCH] Added new UI for the Temporal Sync Feature, made the software more robust including: -Settings window can only be opened once -When the client can't open a device, it tries multiple times -Recording bins are now saved with a unique ID per device, to allow multiple clients to record on the same PC -Synchronized & Unsynchronized devices can be both used in one server -Included a workaround for a bug in the Azure Kinect SDK regarding exposure (microsoft/Azure-Kinect-Sensor-SDK#1378) --- LiveScanPlayer/FrameFileReaderBin.cs | 2 - LiveScanServer/KinectServer.cs | 170 ++++++++++++++++-- LiveScanServer/KinectSettings.cs | 14 +- LiveScanServer/KinectSocket.cs | 98 +++++++++- LiveScanServer/MainWindowForm.cs | 24 ++- LiveScanServer/SettingsForm.Designer.cs | 164 ++++++++++++----- LiveScanServer/SettingsForm.cs | 110 ++++++++++-- include/LiveScanClient/azureKinectCapture.h | 18 +- .../LiveScanClient/frameFileWriterReader.h | 4 +- include/LiveScanClient/iCapture.h | 5 +- include/LiveScanClient/liveScanClient.h | 6 +- include/LiveScanClient/utils.h | 14 +- src/LiveScanClient/azureKinectCapture.cpp | 104 ++++++++++- src/LiveScanClient/frameFileWriterReader.cpp | 8 +- src/LiveScanClient/liveScanClient.cpp | 132 ++++++++++++-- 15 files changed, 744 insertions(+), 129 deletions(-) diff --git a/LiveScanPlayer/FrameFileReaderBin.cs b/LiveScanPlayer/FrameFileReaderBin.cs index 371f46e..8b98ff6 100644 --- a/LiveScanPlayer/FrameFileReaderBin.cs +++ b/LiveScanPlayer/FrameFileReaderBin.cs @@ -46,8 +46,6 @@ namespace LiveScanPlayer lineParts = ReadLine().Split(' '); int frameTimestamp = Int32.Parse(lineParts[1]); - Console.WriteLine((frameTimestamp / 1000).ToString()); - short[] tempVertices = new short[3 * nPoints]; byte[] tempColors = new byte[4 * nPoints]; diff --git a/LiveScanServer/KinectServer.cs b/LiveScanServer/KinectServer.cs index d9c38d7..91adb7d 100644 --- a/LiveScanServer/KinectServer.cs +++ b/LiveScanServer/KinectServer.cs @@ -22,6 +22,7 @@ using System.Threading; using System.Net.Sockets; using System.Net; using System.ComponentModel; +using System.Windows.Forms; namespace KinectServer { @@ -31,9 +32,15 @@ namespace KinectServer Socket oServerSocket; bool bServerRunning = false; - bool waitForSubToStart = false; + bool bWaitForSubToStart = false; + + //This lock prevents the user from enabeling/disabling the Temp Sync State while the cameras are in transition to another state. + //When starting the server, all devices are already initialized, as the LiveScanClient can only connect with an initialized device + bool allDevicesInitialized = true; KinectSettings oSettings; + SettingsForm fSettingsForm; + MainWindowForm fMainWindowForm; object oClientSocketLock = new object(); object oFrameRequestLock = new object(); @@ -133,6 +140,21 @@ namespace KinectServer this.oSettings = settings; } + public void SetSettingsForm(SettingsForm settings) + { + fSettingsForm = settings; + } + + public void SetMainWindowForm(MainWindowForm main) + { + fMainWindowForm = main; + } + + public SettingsForm GetSettingsForm() + { + return fSettingsForm; + } + private void SocketListChanged() { if (eSocketListChanged != null) @@ -226,6 +248,7 @@ namespace KinectServer { lock (oClientSocketLock) { + for (int i = 0; i < lClientSockets.Count; i++) { lClientSockets[i].SendCalibrationData(); @@ -234,33 +257,115 @@ namespace KinectServer } /// - /// Send temporal Sync Data to all clients + /// Request the device Sync state, so that we know which Device is Master and which are Subordinates before starting them + /// + public void RequestDeviceSyncState() + { + lock (oClientSocketLock) + { + for (int i = 0; i < lClientSockets.Count; i++) + { + lClientSockets[i].RequestTempSyncState(); + } + } + } + + /// + /// When a client has send its Device Sync State, we try to Set the Client Sync State /// public void SendTemporalSyncData() { lock (oClientSocketLock) { - if(lClientSockets.Count > 1 && oSettings.bEnableSync) - { - byte syncOffSetCounter = 1; + int masterCount = 0; + int subordinateCount = 0; - for (int i = 0; i < lClientSockets.Count; i++) + //First we check if we have recieved the Device Sync State of all Devices + //If not, we return and the next client who confirms their state starts this function again + for (int i = 0; i < lClientSockets.Count; i++) + { + if(lClientSockets[i].currentDeviceTempSyncState == KinectSocket.eTempSyncConfig.MASTER) + { + masterCount++; + } + + if (lClientSockets[i].currentDeviceTempSyncState == KinectSocket.eTempSyncConfig.SUBORDINATE) + { + subordinateCount++; + } + + if (lClientSockets[i].currentDeviceTempSyncState == KinectSocket.eTempSyncConfig.UNKNOWN) + { + return; + } + } + + //On a second step we check if we have exactly one master and at least one subordinate + + if(masterCount != 1 || subordinateCount < 1) + { + //If not, we show a error message and disable the temporal sync + + fMainWindowForm?.SetStatusBarOnTimer("Temporal Sync cables not connected properly", 5000); + fSettingsForm?.ActivateTempSyncEnableButton(); + return; + } + + + allDevicesInitialized = false; + + byte syncOffSetCounter = 0; + + for (int i = 0; i < lClientSockets.Count; i++) + { + if(lClientSockets[i].currentDeviceTempSyncState == KinectSocket.eTempSyncConfig.SUBORDINATE) { - lClientSockets[i].SendTemporalSyncStatus(true, syncOffSetCounter); syncOffSetCounter++; } - waitForSubToStart = true; + lClientSockets[i].SendTemporalSyncStatus(true, syncOffSetCounter); + } - else + bWaitForSubToStart = true; + } + } + + + /// + /// Sets all clients as standalone + /// + public void DisableTemporalSync() + { + allDevicesInitialized = false; + + lock (oClientSocketLock) + { + for (int i = 0; i < lClientSockets.Count; i++) { - for (int i = 0; i < lClientSockets.Count; i++) + lClientSockets[i].SendTemporalSyncStatus(false, 0); + } + } + } + + + public void ConfirmTemporalSyncDisabled() + { + if (bWaitForSubToStart) + return; + + lock (oClientSocketLock) + { + for (int i = 0; i < lClientSockets.Count; i++) + { + if (lClientSockets[i].currentClientTempSyncState != KinectSocket.eTempSyncConfig.STANDALONE) { - lClientSockets[i].SendTemporalSyncStatus(false, 0); + return; } } } + + allDevicesInitialized = true; } /// @@ -268,7 +373,7 @@ namespace KinectServer /// public void CheckForMasterStart() { - if (!oSettings.bEnableSync || !waitForSubToStart) + if (!bWaitForSubToStart) { return; } @@ -279,23 +384,40 @@ namespace KinectServer { for (int i = 0; i < lClientSockets.Count; i++) { - if (!lClientSockets[i].bSubStarted && lClientSockets[i].currentTempSyncState == KinectSocket.eTempSyncConfig.SUBORDINATE) + if (!lClientSockets[i].bSubStarted && lClientSockets[i].currentClientTempSyncState == KinectSocket.eTempSyncConfig.SUBORDINATE) { allSubsStarted = false; break; } } + bWaitForSubToStart = false; + if (allSubsStarted) { for (int i = 0; i < lClientSockets.Count; i++) { - lClientSockets[i].SendMasterInitialize(); + if(lClientSockets[i].currentDeviceTempSyncState == KinectSocket.eTempSyncConfig.MASTER) + { + lClientSockets[i].SendMasterInitialize(); + return; + } } } - } + } + } - + /// + /// Tells the server that it is now ok to start recieving user changes again + /// + public void MasterSuccessfullyRestarted() + { + allDevicesInitialized = true; + } + + public bool GetAllDevicesInitialized() + { + return allDevicesInitialized; } public bool GetStoredFrame(List> lFramesRGB, List> lFramesVerts) @@ -369,6 +491,7 @@ namespace KinectServer //Wait till frames received bool allGathered = false; + while (!allGathered) { allGathered = true; @@ -384,6 +507,7 @@ namespace KinectServer } } } + } //Store received frames @@ -396,6 +520,7 @@ namespace KinectServer lFramesBody.Add(new List(lClientSockets[i].lBodies)); } } + } } @@ -427,6 +552,9 @@ namespace KinectServer lClientSockets[lClientSockets.Count - 1].SendSettings(oSettings); lClientSockets[lClientSockets.Count - 1].eChanged += new SocketChangedHandler(SocketListChanged); lClientSockets[lClientSockets.Count - 1].eSubInitialized += new SubOrdinateInitialized(CheckForMasterStart); + lClientSockets[lClientSockets.Count - 1].eMasterRestart += new MasterRestarted(MasterSuccessfullyRestarted); + lClientSockets[lClientSockets.Count - 1].eSyncJackstate += new RecievedSyncJackState(SendTemporalSyncData); + lClientSockets[lClientSockets.Count - 1].eStandAloneInitialized += new StandAloneInitialized(ConfirmTemporalSyncDisabled); if (eSocketListChanged != null) { @@ -509,6 +637,16 @@ namespace KinectServer lClientSockets[i].ReceiveTemporalSyncStatus(); } + else if(buffer[0] == 5) + { + lClientSockets[i].RecieveMasterHasRestarted(); + } + + else if(buffer[0] == 6) + { + lClientSockets[i].RecieveDeviceSyncState(); + } + buffer = lClientSockets[i].Receive(1); } } diff --git a/LiveScanServer/KinectSettings.cs b/LiveScanServer/KinectSettings.cs index 987e845..9d42a7e 100644 --- a/LiveScanServer/KinectSettings.cs +++ b/LiveScanServer/KinectSettings.cs @@ -42,7 +42,11 @@ namespace KinectServer public bool bMergeScansForSave = true; public bool bSaveAsBinaryPLY = true; - public bool bEnableSync = false; + public bool bSyncEnabled = false; + + public bool bAutoExposureEnabled = true; + + public int nExposureStep = -8; public KinectSettings() { @@ -102,6 +106,14 @@ namespace KinectServer bTemp = BitConverter.GetBytes(iCompressionLevel); lData.AddRange(bTemp); + if (bAutoExposureEnabled) + lData.Add(1); + else + lData.Add(0); + + bTemp = BitConverter.GetBytes(nExposureStep); + lData.AddRange(bTemp); + return lData; } } diff --git a/LiveScanServer/KinectSocket.cs b/LiveScanServer/KinectSocket.cs index f2fb4a0..5104381 100644 --- a/LiveScanServer/KinectSocket.cs +++ b/LiveScanServer/KinectSocket.cs @@ -18,12 +18,15 @@ using System.Linq; using System.Text; using System.Threading.Tasks; using System.Net.Sockets; - +using OpenTK.Graphics; namespace KinectServer { public delegate void SocketChangedHandler(); public delegate void SubOrdinateInitialized(); + public delegate void MasterRestarted(); + public delegate void RecievedSyncJackState(); + public delegate void StandAloneInitialized(); public class KinectSocket { @@ -35,11 +38,19 @@ namespace KinectServer public bool bNoMoreStoredFrames = true; public bool bCalibrated = false; public bool bSubStarted = false; + public bool bMasterStarted = false; + + public enum eTempSyncConfig { MASTER, SUBORDINATE, STANDALONE, UNKNOWN } + + //Shows how the *Device* is configured (Determined only by the Sync-Cables hooked up to the device) + public eTempSyncConfig currentDeviceTempSyncState = eTempSyncConfig.STANDALONE; + + //Shows how the Client-Software is configured (This is set by the server) + public eTempSyncConfig currentClientTempSyncState = eTempSyncConfig.STANDALONE; - public enum eTempSyncConfig { MASTER, SUBORDINATE, STANDALONE } - public eTempSyncConfig currentTempSyncState = eTempSyncConfig.STANDALONE; //The pose of the sensor in the scene (used by the OpenGLWindow to show the sensor) public AffineTransform oCameraPose = new AffineTransform(); + //The transform that maps the vertices in the sensor coordinate system to the world corrdinate system. public AffineTransform oWorldTransform = new AffineTransform(); @@ -51,6 +62,9 @@ namespace KinectServer public event SocketChangedHandler eChanged; public event SubOrdinateInitialized eSubInitialized; + public event MasterRestarted eMasterRestart; + public event RecievedSyncJackState eSyncJackstate; + public event StandAloneInitialized eStandAloneInitialized; public KinectSocket(Socket clientSocket) { @@ -135,7 +149,10 @@ namespace KinectServer /// Enable/Disable Temporal Sync /// Only set when this device should be a subordinate. Should be a unique number in ascending order for each sub public void SendTemporalSyncStatus(bool tempSyncOn, byte syncOffSetMultiplier) - { + { + bSubStarted = false; + currentClientTempSyncState = eTempSyncConfig.UNKNOWN; + if (tempSyncOn) { byte[] data = new byte[2]; @@ -161,6 +178,49 @@ namespace KinectServer SendByte(); } + + /// + /// Asks the client in which way the Sync-Cables are hooked up to the device + /// + public void RequestTempSyncState() + { + currentDeviceTempSyncState = eTempSyncConfig.UNKNOWN; + byteToSend[0] = 10; + SendByte(); + } + + public void RecieveDeviceSyncState() + { + byte tempSyncState; + byte[] buffer = new byte[1024]; + + while (oSocket.Available == 0) + { + if (!SocketConnected()) + return; + } + + oSocket.Receive(buffer, 3, SocketFlags.None); + tempSyncState = buffer[0]; + + if (tempSyncState == 0) + { + currentDeviceTempSyncState = eTempSyncConfig.SUBORDINATE; + } + + else if (tempSyncState == 1) + { + currentDeviceTempSyncState = eTempSyncConfig.MASTER; + } + + else if (tempSyncState == 2) + { + currentDeviceTempSyncState = eTempSyncConfig.STANDALONE; + } + + eSyncJackstate.Invoke(); + } + public void ReceiveCalibrationData() { bCalibrated = true; @@ -205,25 +265,36 @@ namespace KinectServer if (tempSyncState == 0) { - currentTempSyncState = eTempSyncConfig.SUBORDINATE; + currentClientTempSyncState = eTempSyncConfig.SUBORDINATE; bSubStarted = true; + bMasterStarted = false; eSubInitialized?.Invoke(); } else if(tempSyncState == 1) { - currentTempSyncState = eTempSyncConfig.MASTER; + currentClientTempSyncState = eTempSyncConfig.MASTER; + bSubStarted = false; } else if (tempSyncState == 2) { - currentTempSyncState = eTempSyncConfig.STANDALONE; + currentClientTempSyncState = eTempSyncConfig.STANDALONE; + bSubStarted = false; + bMasterStarted = false; + eStandAloneInitialized?.Invoke(); } UpdateSocketState(); } + public void RecieveMasterHasRestarted() + { + bMasterStarted = true; + eMasterRestart?.Invoke(); + } + public void ReceiveFrame() { lFrameRGB.Clear(); @@ -241,6 +312,9 @@ namespace KinectServer oSocket.Receive(buffer, 8, SocketFlags.None); nToRead = BitConverter.ToInt32(buffer, 0); + + + int iCompressed = BitConverter.ToInt32(buffer, 4); if (nToRead == -1) @@ -249,6 +323,14 @@ namespace KinectServer return; } + //Sometimes we recieve negative values when the cameras are restarting, I don't know why yet. + //I'm just patching this up for now.s + + if (nToRead <= 0) + { + return; + } + buffer = new byte[nToRead]; int nAlreadyRead = 0; @@ -371,7 +453,7 @@ namespace KinectServer { string tempSyncMessage = ""; - switch (currentTempSyncState) + switch (currentClientTempSyncState) { case eTempSyncConfig.MASTER: tempSyncMessage = "[MASTER]"; diff --git a/LiveScanServer/MainWindowForm.cs b/LiveScanServer/MainWindowForm.cs index 7b7fdc2..09ad823 100644 --- a/LiveScanServer/MainWindowForm.cs +++ b/LiveScanServer/MainWindowForm.cs @@ -75,8 +75,6 @@ namespace KinectServer IFormatter formatter = new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter(); Stream stream = new FileStream("settings.bin", FileMode.Open, FileAccess.Read); oSettings = (KinectSettings)formatter.Deserialize(stream); - //This always needs to be set to false on startup, as we can only activate it when all Kinects are already connected - oSettings.bEnableSync = false; stream.Close(); } catch(Exception) @@ -127,10 +125,16 @@ namespace KinectServer //Opens the settings form private void btSettings_Click(object sender, EventArgs e) { - SettingsForm form = new SettingsForm(); - form.oSettings = oSettings; - form.oServer = oServer; - form.Show(); + if(oServer.GetSettingsForm() == null) + { + SettingsForm form = new SettingsForm(); + form.oSettings = oSettings; + form.oServer = oServer; + form.Show(); + oServer.SetSettingsForm(form); + oServer.SetMainWindowForm(this); + } + } //Performs recording which is synchronized frame capture. @@ -265,6 +269,12 @@ namespace KinectServer while (!worker.CancellationPending) { Thread.Sleep(1); + + if (!oServer.GetAllDevicesInitialized()) + { + continue; + } + oServer.GetLatestFrame(lFramesRGB, lFramesVerts, lFramesBody); //Update the vertex and color lists that are common between this class and the OpenGLWindow. @@ -474,7 +484,7 @@ namespace KinectServer OpenGLWorker.RunWorkerAsync(); } - private void SetStatusBarOnTimer(string message, int milliseconds) + public void SetStatusBarOnTimer(string message, int milliseconds) { statusLabel.Text = message; diff --git a/LiveScanServer/SettingsForm.Designer.cs b/LiveScanServer/SettingsForm.Designer.cs index 0387c58..99358a5 100644 --- a/LiveScanServer/SettingsForm.Designer.cs +++ b/LiveScanServer/SettingsForm.Designer.cs @@ -33,9 +33,15 @@ this.lbICPIters = new System.Windows.Forms.Label(); this.txtICPIters = new System.Windows.Forms.TextBox(); this.grClient = new System.Windows.Forms.GroupBox(); - this.grSync = new System.Windows.Forms.GroupBox(); + this.grExposure = new System.Windows.Forms.GroupBox(); + this.lbManualExposure = new System.Windows.Forms.Label(); + this.chAutoExposureEnabled = new System.Windows.Forms.CheckBox(); + this.trManualExposure = new System.Windows.Forms.TrackBar(); + this.grTempSync = new System.Windows.Forms.GroupBox(); + this.btSyncDisable = new System.Windows.Forms.Button(); + this.btSyncEnable = new System.Windows.Forms.Button(); + this.lbTempSync = new System.Windows.Forms.Label(); this.label1 = new System.Windows.Forms.Label(); - this.chSync = new System.Windows.Forms.CheckBox(); this.grBody = new System.Windows.Forms.GroupBox(); this.chSkeletons = new System.Windows.Forms.CheckBox(); this.chBodyData = new System.Windows.Forms.CheckBox(); @@ -82,9 +88,10 @@ this.rAsciiPly = new System.Windows.Forms.RadioButton(); this.txtRefinIters = new System.Windows.Forms.TextBox(); this.lbOuterIters = new System.Windows.Forms.Label(); - this.label3 = new System.Windows.Forms.Label(); this.grClient.SuspendLayout(); - this.grSync.SuspendLayout(); + this.grExposure.SuspendLayout(); + ((System.ComponentModel.ISupportInitialize)(this.trManualExposure)).BeginInit(); + this.grTempSync.SuspendLayout(); this.grBody.SuspendLayout(); this.grMarkers.SuspendLayout(); this.grBounding.SuspendLayout(); @@ -131,30 +138,108 @@ // // grClient // - this.grClient.Controls.Add(this.grSync); + this.grClient.Controls.Add(this.grExposure); + this.grClient.Controls.Add(this.grTempSync); this.grClient.Controls.Add(this.grBody); this.grClient.Controls.Add(this.grMarkers); this.grClient.Controls.Add(this.grBounding); this.grClient.Controls.Add(this.grFiltering); this.grClient.Location = new System.Drawing.Point(12, 12); this.grClient.Name = "grClient"; - this.grClient.Size = new System.Drawing.Size(632, 291); + this.grClient.Size = new System.Drawing.Size(632, 408); this.grClient.TabIndex = 43; this.grClient.TabStop = false; this.grClient.Text = "KinectClient settings"; // - // grSync + // grExposure // - this.grSync.Controls.Add(this.label3); - this.grSync.Controls.Add(this.label1); - this.grSync.Controls.Add(this.chSync); - this.grSync.Location = new System.Drawing.Point(271, 211); - this.grSync.Name = "grSync"; - this.grSync.Size = new System.Drawing.Size(355, 80); - this.grSync.TabIndex = 48; - this.grSync.TabStop = false; - this.grSync.Text = "Temporal Sync"; - this.grSync.Enter += new System.EventHandler(this.grSync_Enter); + this.grExposure.Controls.Add(this.lbManualExposure); + this.grExposure.Controls.Add(this.chAutoExposureEnabled); + this.grExposure.Controls.Add(this.trManualExposure); + this.grExposure.Location = new System.Drawing.Point(9, 283); + this.grExposure.Name = "grExposure"; + this.grExposure.Size = new System.Drawing.Size(249, 115); + this.grExposure.TabIndex = 49; + this.grExposure.TabStop = false; + this.grExposure.Text = "Exposure Settings"; + // + // lbManualExposure + // + this.lbManualExposure.AutoSize = true; + this.lbManualExposure.Location = new System.Drawing.Point(79, 52); + this.lbManualExposure.Name = "lbManualExposure"; + this.lbManualExposure.Size = new System.Drawing.Size(91, 13); + this.lbManualExposure.TabIndex = 7; + this.lbManualExposure.Text = "Manual exposure:"; + // + // chAutoExposureEnabled + // + this.chAutoExposureEnabled.AutoSize = true; + this.chAutoExposureEnabled.Checked = true; + this.chAutoExposureEnabled.CheckState = System.Windows.Forms.CheckState.Checked; + this.chAutoExposureEnabled.Location = new System.Drawing.Point(11, 20); + this.chAutoExposureEnabled.Name = "chAutoExposureEnabled"; + this.chAutoExposureEnabled.Size = new System.Drawing.Size(135, 17); + this.chAutoExposureEnabled.TabIndex = 6; + this.chAutoExposureEnabled.Text = "Auto exposure enabled"; + this.chAutoExposureEnabled.UseVisualStyleBackColor = true; + this.chAutoExposureEnabled.CheckedChanged += new System.EventHandler(this.chAutoExposureEnabled_CheckedChanged); + // + // trManualExposure + // + this.trManualExposure.Enabled = false; + this.trManualExposure.LargeChange = 1; + this.trManualExposure.Location = new System.Drawing.Point(6, 62); + this.trManualExposure.Maximum = -5; + this.trManualExposure.Minimum = -11; + this.trManualExposure.Name = "trManualExposure"; + this.trManualExposure.Size = new System.Drawing.Size(237, 45); + this.trManualExposure.TabIndex = 5; + this.trManualExposure.Value = -8; + this.trManualExposure.Scroll += new System.EventHandler(this.trManualExposure_Scroll); + // + // grTempSync + // + this.grTempSync.Controls.Add(this.btSyncDisable); + this.grTempSync.Controls.Add(this.btSyncEnable); + this.grTempSync.Controls.Add(this.lbTempSync); + this.grTempSync.Controls.Add(this.label1); + this.grTempSync.Location = new System.Drawing.Point(271, 211); + this.grTempSync.Name = "grTempSync"; + this.grTempSync.Size = new System.Drawing.Size(355, 97); + this.grTempSync.TabIndex = 48; + this.grTempSync.TabStop = false; + this.grTempSync.Text = "Temporal Sync"; + // + // btSyncDisable + // + this.btSyncDisable.Location = new System.Drawing.Point(104, 19); + this.btSyncDisable.Name = "btSyncDisable"; + this.btSyncDisable.Size = new System.Drawing.Size(92, 23); + this.btSyncDisable.TabIndex = 4; + this.btSyncDisable.Text = "Disable"; + this.btSyncDisable.UseVisualStyleBackColor = true; + this.btSyncDisable.Click += new System.EventHandler(this.btSyncDisable_click); + // + // btSyncEnable + // + this.btSyncEnable.Location = new System.Drawing.Point(6, 19); + this.btSyncEnable.Name = "btSyncEnable"; + this.btSyncEnable.Size = new System.Drawing.Size(92, 23); + this.btSyncEnable.TabIndex = 3; + this.btSyncEnable.Text = "Enable"; + this.btSyncEnable.UseVisualStyleBackColor = true; + this.btSyncEnable.Click += new System.EventHandler(this.btSyncEnable_click); + // + // lbTempSync + // + this.lbTempSync.Location = new System.Drawing.Point(3, 45); + this.lbTempSync.Name = "lbTempSync"; + this.lbTempSync.Size = new System.Drawing.Size(342, 47); + this.lbTempSync.TabIndex = 2; + this.lbTempSync.Text = "Before activating, make sure that all Kinects are connected to the server and pro" + + "perly connected via the sync cables. When Temporal Sync is activated, the exposu" + + "re needs to be set manually!"; // // label1 // @@ -166,17 +251,6 @@ this.label1.TabIndex = 1; this.label1.Text = "Notice: All devices need to be connected to the server and via sync cabel before" + " enabeling! First Device in Server List will become the master"; - // - // chSync - // - this.chSync.AutoSize = true; - this.chSync.Location = new System.Drawing.Point(6, 19); - this.chSync.Name = "chSync"; - this.chSync.Size = new System.Drawing.Size(133, 17); - this.chSync.TabIndex = 0; - this.chSync.Text = "Enable Temporal Sync"; - this.chSync.UseVisualStyleBackColor = true; - this.chSync.CheckedChanged += new System.EventHandler(this.chSync_CheckedChanged); // // grBody // @@ -192,6 +266,7 @@ // chSkeletons // this.chSkeletons.AutoSize = true; + this.chSkeletons.Enabled = false; this.chSkeletons.Location = new System.Drawing.Point(11, 41); this.chSkeletons.Name = "chSkeletons"; this.chSkeletons.Size = new System.Drawing.Size(154, 17); @@ -203,6 +278,7 @@ // chBodyData // this.chBodyData.AutoSize = true; + this.chBodyData.Enabled = false; this.chBodyData.Location = new System.Drawing.Point(11, 19); this.chBodyData.Name = "chBodyData"; this.chBodyData.Size = new System.Drawing.Size(113, 17); @@ -560,7 +636,7 @@ this.grServer.Controls.Add(this.txtICPIters); this.grServer.Controls.Add(this.chMerge); this.grServer.Controls.Add(this.lbICPIters); - this.grServer.Location = new System.Drawing.Point(12, 311); + this.grServer.Location = new System.Drawing.Point(12, 426); this.grServer.Name = "grServer"; this.grServer.Size = new System.Drawing.Size(632, 114); this.grServer.TabIndex = 44; @@ -649,32 +725,25 @@ this.lbOuterIters.Size = new System.Drawing.Size(118, 13); this.lbOuterIters.TabIndex = 26; this.lbOuterIters.Text = "Num of refinement iters:"; - // - // label3 - // - this.label3.Location = new System.Drawing.Point(7, 36); - this.label3.Name = "label3"; - this.label3.Size = new System.Drawing.Size(342, 44); - this.label3.TabIndex = 2; - this.label3.Text = "Before activating, make sure that all Kinects are connected to the server and pro" + - "perly connected via the sync cables. The first device in the server list will be" + - "come the master!"; // // SettingsForm // this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - this.ClientSize = new System.Drawing.Size(656, 431); + this.ClientSize = new System.Drawing.Size(656, 554); this.Controls.Add(this.grServer); this.Controls.Add(this.grClient); this.FormBorderStyle = System.Windows.Forms.FormBorderStyle.FixedSingle; this.MaximizeBox = false; this.Name = "SettingsForm"; this.Text = "Settings"; + this.FormClosed += new System.Windows.Forms.FormClosedEventHandler(this.SettingsForm_FormClosed); this.Load += new System.EventHandler(this.SettingsForm_Load); this.grClient.ResumeLayout(false); - this.grSync.ResumeLayout(false); - this.grSync.PerformLayout(); + this.grExposure.ResumeLayout(false); + this.grExposure.PerformLayout(); + ((System.ComponentModel.ISupportInitialize)(this.trManualExposure)).EndInit(); + this.grTempSync.ResumeLayout(false); this.grBody.ResumeLayout(false); this.grBody.PerformLayout(); this.grMarkers.ResumeLayout(false); @@ -742,9 +811,14 @@ private System.Windows.Forms.CheckBox chSkeletons; private System.Windows.Forms.ComboBox cbCompressionLevel; private System.Windows.Forms.Label label2; - private System.Windows.Forms.GroupBox grSync; - private System.Windows.Forms.CheckBox chSync; + private System.Windows.Forms.GroupBox grTempSync; private System.Windows.Forms.Label label1; - private System.Windows.Forms.Label label3; + private System.Windows.Forms.Label lbTempSync; + private System.Windows.Forms.Button btSyncDisable; + private System.Windows.Forms.Button btSyncEnable; + private System.Windows.Forms.TrackBar trManualExposure; + private System.Windows.Forms.GroupBox grExposure; + private System.Windows.Forms.Label lbManualExposure; + private System.Windows.Forms.CheckBox chAutoExposureEnabled; } } \ No newline at end of file diff --git a/LiveScanServer/SettingsForm.cs b/LiveScanServer/SettingsForm.cs index b7a0665..0158fe0 100644 --- a/LiveScanServer/SettingsForm.cs +++ b/LiveScanServer/SettingsForm.cs @@ -30,6 +30,8 @@ namespace KinectServer public KinectSettings oSettings; public KinectServer oServer; + private Timer scrollTimer = null; + bool bFormLoaded = false; public SettingsForm() @@ -62,7 +64,12 @@ namespace KinectServer txtICPIters.Text = oSettings.nNumICPIterations.ToString(); txtRefinIters.Text = oSettings.nNumRefineIters.ToString(); - chSync.Checked = oSettings.bEnableSync; + btSyncEnable.Enabled = true; + btSyncDisable.Enabled = false; + + chAutoExposureEnabled.Checked = oSettings.bAutoExposureEnabled; + + trManualExposure.Value = oSettings.nExposureStep; if (oSettings.bSaveAsBinaryPLY) { @@ -83,7 +90,6 @@ namespace KinectServer if (bFormLoaded) { oServer.SendSettings(); - oServer.SendTemporalSyncData(); } } @@ -348,21 +354,105 @@ namespace KinectServer UpdateClients(); } - - private void chSync_CheckedChanged(object sender, EventArgs e) + private void btSyncEnable_click(object sender, EventArgs e) { - oSettings.bEnableSync = chSync.Checked; + if (oServer.GetAllDevicesInitialized() && oServer.nClientCount > 1) + { + oServer.RequestDeviceSyncState(); + btSyncEnable.Enabled = false; + btSyncDisable.Enabled = true; + + //Disable the Auto Exposure, as this could interfere with the temporal sync + chAutoExposureEnabled.Enabled = false; + trManualExposure.Enabled = true; + chAutoExposureEnabled.CheckState = CheckState.Unchecked; + } + + } + + private void btSyncDisable_click(object sender, EventArgs e) + { + if (oServer.GetAllDevicesInitialized()) + { + oServer.DisableTemporalSync(); + btSyncEnable.Enabled = true; + btSyncDisable.Enabled = false; + chAutoExposureEnabled.Enabled = true; + } + } + + public void ActivateTempSyncEnableButton() + { + //As a Form code is run on a single UI thread, we need to communicate with that thread via Invoke + this.BeginInvoke(new MethodInvoker(delegate + { + btSyncEnable.Enabled = true; + btSyncDisable.Enabled = false; + chAutoExposureEnabled.Enabled = true; + })); + } + + private void chAutoExposureEnabled_CheckedChanged(object sender, EventArgs e) + { + oSettings.bAutoExposureEnabled = chAutoExposureEnabled.Checked; UpdateClients(); + + trManualExposure.Enabled = !oSettings.bAutoExposureEnabled; } - private void grSync_Enter(object sender, EventArgs e) - { + /// + /// When the user scrolls on the trackbar, we wait a short amount of time to check if the user has scrolled again. + /// This prevents the Manual Exposure to be set too often, and only sets it when the user has stopped scrolling. + /// Code taken from: https://stackoverflow.com/a/15687418 + /// + /// + /// + private void trManualExposure_Scroll(object sender, EventArgs e) + { + if (scrollTimer == null) + { + // Will tick every 500ms + scrollTimer = new Timer() + { + Enabled = false, + Interval = 300, + Tag = (sender as TrackBar).Value + }; + + scrollTimer.Tick += (s, ea) => + { + // check to see if the value has changed since we last ticked + if (trManualExposure.Value == (int)scrollTimer.Tag) + { + // scrolling has stopped so we are good to go ahead and do stuff + scrollTimer.Stop(); + + // Send the changed exposure to the devices + + //Clamp Exposure Step between -11 and 1 + int exposureStep = trManualExposure.Value; + int exposureStepClamped = exposureStep < -11 ? -11 : exposureStep > 1 ? 1 : exposureStep; + oSettings.nExposureStep = exposureStepClamped; + UpdateClients(); + + + scrollTimer.Dispose(); + scrollTimer = null; + } + else + { + // record the last value seen + scrollTimer.Tag = trManualExposure.Value; + } + }; + scrollTimer.Start(); + } } - private void lbSyncWarning_Click(object sender, EventArgs e) + private void SettingsForm_FormClosed(object sender, FormClosedEventArgs e) { - + oServer.SetSettingsForm(null); } } -} +} \ No newline at end of file diff --git a/include/LiveScanClient/azureKinectCapture.h b/include/LiveScanClient/azureKinectCapture.h index b1ccda3..1a99141 100644 --- a/include/LiveScanClient/azureKinectCapture.h +++ b/include/LiveScanClient/azureKinectCapture.h @@ -3,7 +3,9 @@ #include "stdafx.h" #include "ICapture.h" #include -#include "utils.h" +#include +using namespace std; +//#include "utils.h" class AzureKinectCapture : public ICapture { @@ -11,15 +13,17 @@ public: AzureKinectCapture(); ~AzureKinectCapture(); - bool Initialize(bool asMaster, bool asSubOrdinate, int syncOffset); - bool Close(); + bool Initialize(SYNC_STATE state, int syncOffset); bool AcquireFrame(); + bool Close(); void MapDepthFrameToCameraSpace(Point3f *pCameraSpacePoints); void MapColorFrameToCameraSpace(Point3f *pCameraSpacePoints); void MapDepthFrameToColorSpace(UINT16 *pDepthInColorSpace); void MapColorFrameToDepthSpace(RGB *pColorInDepthSpace); int GetSyncJackState(); uint64_t GetTimeStamp(); + int GetDeviceIndex(); + void SetExposureState(bool enableAutoExposure, int exposureStep); private: @@ -34,6 +38,12 @@ private: k4a_transformation_t transformation = NULL; bool syncInConnected = false; bool syncOutConnected = false; - uint64_t currentTimeStamp; + uint64_t currentTimeStamp = 0; + SYNC_STATE syncState = Standalone; + int deviceIDForRestart = -1; + int restartAttempts = 0; + bool autoExposureEnabled = true; + int exposureTimeStep = 0; void UpdateDepthPointCloud(); + }; \ No newline at end of file diff --git a/include/LiveScanClient/frameFileWriterReader.h b/include/LiveScanClient/frameFileWriterReader.h index 5f80668..b5f7a3f 100644 --- a/include/LiveScanClient/frameFileWriterReader.h +++ b/include/LiveScanClient/frameFileWriterReader.h @@ -10,13 +10,13 @@ class FrameFileWriterReader { public: FrameFileWriterReader(); - void openNewFileForWriting(); + void openNewFileForWriting(int deviceID); void openCurrentFileForReading(); // leave filename blank if you want the filename to be generated from the date void setCurrentFilename(std::string filename = ""); - void writeFrame(std::vector points, std::vector colors, uint64_t timestamp); + void writeFrame(std::vector points, std::vector colors, uint64_t timestamp, int deviceID); bool readFrame(std::vector &outPoints, std::vector &outColors); bool openedForWriting() { return m_bFileOpenedForWriting; } diff --git a/include/LiveScanClient/iCapture.h b/include/LiveScanClient/iCapture.h index a0bf2e8..afe0ca5 100644 --- a/include/LiveScanClient/iCapture.h +++ b/include/LiveScanClient/iCapture.h @@ -40,7 +40,7 @@ public: ICapture(); ~ICapture(); - virtual bool Initialize(bool asMaster, bool asSubOrdinate, int syncOffset) = 0; + virtual bool Initialize(SYNC_STATE state, int syncOffset) = 0; virtual bool AcquireFrame() = 0; virtual bool Close() = 0; virtual void MapDepthFrameToCameraSpace(Point3f *pCameraSpacePoints) = 0; @@ -49,6 +49,9 @@ public: virtual void MapColorFrameToDepthSpace(RGB *pDepthSpacePoints) = 0; virtual int GetSyncJackState() = 0; virtual uint64_t GetTimeStamp() = 0; + virtual int GetDeviceIndex() = 0; + virtual void SetExposureState(bool enableAutoExposure, int exposureStep) = 0; + bool bInitialized; diff --git a/include/LiveScanClient/liveScanClient.h b/include/LiveScanClient/liveScanClient.h index 41f550f..99be1f3 100644 --- a/include/LiveScanClient/liveScanClient.h +++ b/include/LiveScanClient/liveScanClient.h @@ -18,7 +18,7 @@ #include "ImageRenderer.h" #include "SocketCS.h" #include "calibration.h" -#include "utils.h" +//#include "utils.h" #include "azureKinectCapture.h" #include "frameFileWriterReader.h" #include @@ -45,6 +45,7 @@ private: bool m_bIsMaster; bool m_bIsSubOrdinate; + bool m_bRestartingCamera; ICapture *pCapture; @@ -56,10 +57,13 @@ private: bool m_bConfirmCaptured; bool m_bConfirmTempSyncState; bool m_bConfirmSubOrdinateStarted; + bool m_bConfirmRestartAsMaster; bool m_bConfirmCalibrated; bool m_bShowDepth; bool m_bFrameCompression; int m_iCompressionLevel; + bool m_bAutoExposureEnabled; + int m_nExposureStep; enum tempSyncConfig { MASTER, SUBORDINATE, STANDALONE }; tempSyncConfig currentTempSyncState; diff --git a/include/LiveScanClient/utils.h b/include/LiveScanClient/utils.h index a6eeed9..e7918e7 100644 --- a/include/LiveScanClient/utils.h +++ b/include/LiveScanClient/utils.h @@ -30,7 +30,8 @@ enum INCOMING_MESSAGE_TYPE MSG_CLEAR_STORED_FRAMES, MSG_SET_TEMPSYNC_ON, MSG_SET_TEMPSYNC_OFF, - MSG_START_MASTER + MSG_START_MASTER, + MSG_REQUEST_SYNC_JACK_STATE, }; enum OUTGOING_MESSAGE_TYPE @@ -39,7 +40,16 @@ enum OUTGOING_MESSAGE_TYPE MSG_CONFIRM_CALIBRATED, MSG_STORED_FRAME, MSG_LAST_FRAME, - MSG_CONFIRM_TEMP_SYNC_STATUS + MSG_CONFIRM_TEMP_SYNC_STATUS, + MSG_CONFIRM_MASTER_RESTART, + MSG_SYNC_JACK_STATE +}; + +enum SYNC_STATE +{ + Subordinate, + Master, + Standalone }; typedef struct Point3f diff --git a/src/LiveScanClient/azureKinectCapture.cpp b/src/LiveScanClient/azureKinectCapture.cpp index a69a291..8d7a4d4 100644 --- a/src/LiveScanClient/azureKinectCapture.cpp +++ b/src/LiveScanClient/azureKinectCapture.cpp @@ -24,20 +24,51 @@ AzureKinectCapture::~AzureKinectCapture() /// Initalizes this devices as subordinate /// Should only be set when initializing as Subordinate. Each subordinate should have a unique, ascending value /// -bool AzureKinectCapture::Initialize(bool asMaster, bool asSubordinate, int syncOffsetMultiplier) +bool AzureKinectCapture::Initialize(SYNC_STATE state, int syncOffsetMultiplier) { uint32_t count = k4a_device_get_installed_count(); int deviceIdx = 0; + //We save the deviceId of this Client. + //When the cameras are reinitialized during runtime, we can then gurantee + //that each LiveScan instance uses the same device as before (In case two or more Kinects are connected to the same PC) + //A device ID of -1 means that no Kinects has been successfully initalized yet (only happens when the Client starts) + if (deviceIDForRestart != -1) { + deviceIdx = deviceIDForRestart; + } + kinectSensor = NULL; while (K4A_FAILED(k4a_device_open(deviceIdx, &kinectSensor))) { - deviceIdx++; - if (deviceIdx >= count) + + if (deviceIDForRestart == -1) { - bInitialized = false; - return bInitialized; + deviceIdx++; + if (deviceIdx >= count) + { + bInitialized = false; + return bInitialized; + } } + + + else { + + //Sometimes the cameras fail to reinitialize, so we try to initialize them three times with a slight delay before failing + if (restartAttempts > 2) + { + bInitialized = false; + return bInitialized; + } + + else { + + restartAttempts++; + Sleep(200); + } + + } + } k4a_device_configuration_t config = K4A_DEVICE_CONFIG_INIT_DISABLE_ALL; @@ -47,12 +78,12 @@ bool AzureKinectCapture::Initialize(bool asMaster, bool asSubordinate, int syncO config.depth_mode = K4A_DEPTH_MODE_NFOV_UNBINNED; config.synchronized_images_only = true; - if (asMaster) + if (state == Master) { config.wired_sync_mode = K4A_WIRED_SYNC_MODE_MASTER; } - else if (asSubordinate) + else if (state == Subordinate) { config.wired_sync_mode = K4A_WIRED_SYNC_MODE_SUBORDINATE; //Sets the offset on subordinate devices. Should be a multiple of 160, each subordinate having a different multiplier in ascending order. @@ -75,10 +106,24 @@ bool AzureKinectCapture::Initialize(bool asMaster, bool asSubordinate, int syncO return bInitialized; } + //Workaround for a bug. When the camera starts in manual Exposure mode, the brightness of the RBG image + //is much lower than in auto exposure mode. To prevent this, we first set the camera to auto exposure mode and + //then switch to manual mode again if it has been enabled before + + if (autoExposureEnabled == false) { + + k4a_device_set_color_control(kinectSensor, K4A_COLOR_CONTROL_EXPOSURE_TIME_ABSOLUTE, K4A_COLOR_CONTROL_MODE_AUTO, 0); + + //Give it a second to adjust + Sleep(1000); + + SetExposureState(false, exposureTimeStep); + } + transformation = k4a_transformation_create(&calibration); //If this device is a subordinate, it is expected to start capturing at a later time (When the master has started), so we skip this check - if (!asSubordinate) + if (state != Subordinate) { std::chrono::time_point start = std::chrono::system_clock::now(); bool bTemp; @@ -100,6 +145,9 @@ bool AzureKinectCapture::Initialize(bool asMaster, bool asSubordinate, int syncO serialNumber = std::string(serialNoSize, '\0'); k4a_device_get_serialnum(kinectSensor, (char*)serialNumber.c_str(), &serialNoSize); + deviceIDForRestart = deviceIdx; + restartAttempts = 0; + return bInitialized; } @@ -173,6 +221,41 @@ bool AzureKinectCapture::AcquireFrame() return true; } +/// +/// Enables/Disables Auto Exposure and/or sets the exposure to a step value between -11 and -5 +/// The kinect supports exposure values up to 1, but these are only available in lower FPS modes (5 or 15 FPS) +/// For further information on how this value translates to ms-Values, please take a look +/// at this table: https://github.com/microsoft/Azure-Kinect-Sensor-SDK/blob/develop/src/color/color_priv.h +/// +/// The Exposure Step between -11 and 1 +void AzureKinectCapture::SetExposureState(bool enableAutoExposure, int exposureStep) +{ + if (bInitialized) + { + if (enableAutoExposure) + { + k4a_device_set_color_control(kinectSensor, K4A_COLOR_CONTROL_EXPOSURE_TIME_ABSOLUTE, K4A_COLOR_CONTROL_MODE_AUTO, 0); + + autoExposureEnabled = true; + } + + else + { + //Formula copied from here: https://github.com/microsoft/Azure-Kinect-Sensor-SDK/blob/7cd8683a1a71b8baebef4a3537e6edd8639d1e95/examples/k4arecorder/main.cpp#L333 + float absExposure = (exp2f((float)exposureStep) * 1000000.0f); + //We add 0.5 because C++ always truncates when converting to an integer. + //This ensures that values will always be rounded correctly + float absExposureRoundingMargin = absExposure + 0.5; + int32_t absoluteExposureInt = (int32_t)absExposureRoundingMargin; + + k4a_device_set_color_control(kinectSensor, K4A_COLOR_CONTROL_EXPOSURE_TIME_ABSOLUTE, K4A_COLOR_CONTROL_MODE_MANUAL, absoluteExposureInt); + + autoExposureEnabled = false; + exposureTimeStep = exposureStep; + } + } +} + void AzureKinectCapture::UpdateDepthPointCloud() { if (depthPointCloudImage == NULL) @@ -348,3 +431,8 @@ uint64_t AzureKinectCapture::GetTimeStamp() return currentTimeStamp; } +int AzureKinectCapture::GetDeviceIndex() +{ + return deviceIDForRestart; +} + diff --git a/src/LiveScanClient/frameFileWriterReader.cpp b/src/LiveScanClient/frameFileWriterReader.cpp index c1a4f7f..eb52a13 100644 --- a/src/LiveScanClient/frameFileWriterReader.cpp +++ b/src/LiveScanClient/frameFileWriterReader.cpp @@ -39,14 +39,14 @@ void FrameFileWriterReader::openCurrentFileForReading() m_bFileOpenedForWriting = false; } -void FrameFileWriterReader::openNewFileForWriting() +void FrameFileWriterReader::openNewFileForWriting(int deviceID) { closeFileIfOpened(); char filename[1024]; time_t t = time(0); struct tm * now = localtime(&t); - sprintf(filename, "recording_%04d_%02d_%02d_%02d_%02d.bin", now->tm_year + 1900, now->tm_mon + 1, now->tm_mday, now->tm_hour, now->tm_min, now->tm_sec); + sprintf(filename, "recording_%01d_%04d_%02d_%02d_%02d_%02d.bin", deviceID, now->tm_year + 1900, now->tm_mon + 1, now->tm_mday, now->tm_hour, now->tm_min, now->tm_sec); m_sFilename = filename; m_pFileHandle = fopen(filename, "wb"); @@ -86,10 +86,10 @@ bool FrameFileWriterReader::readFrame(std::vector &outPoints, std::vect } -void FrameFileWriterReader::writeFrame(std::vector points, std::vector colors, uint64_t timestamp) +void FrameFileWriterReader::writeFrame(std::vector points, std::vector colors, uint64_t timestamp, int deviceID) { if (!m_bFileOpenedForWriting) - openNewFileForWriting(); + openNewFileForWriting(deviceID); FILE *f = m_pFileHandle; diff --git a/src/LiveScanClient/liveScanClient.cpp b/src/LiveScanClient/liveScanClient.cpp index 19c2c9c..36eb510 100644 --- a/src/LiveScanClient/liveScanClient.cpp +++ b/src/LiveScanClient/liveScanClient.cpp @@ -38,13 +38,13 @@ int APIENTRY wWinMain( } LiveScanClient::LiveScanClient() : - m_hWnd(NULL), - m_nLastCounter(0), - m_nFramesSinceUpdate(0), - m_fFreq(0), - m_nNextStatusTime(0LL), - m_pD2DFactory(NULL), - m_pDrawColor(NULL), + m_hWnd(NULL), + m_nLastCounter(0), + m_nFramesSinceUpdate(0), + m_fFreq(0), + m_nNextStatusTime(0LL), + m_pD2DFactory(NULL), + m_pDrawColor(NULL), m_pDepthRGBX(NULL), m_pCameraSpaceCoordinates(NULL), m_pColorInDepthSpace(NULL), @@ -56,13 +56,17 @@ LiveScanClient::LiveScanClient() : m_bConnected(false), m_bConfirmCaptured(false), m_bConfirmCalibrated(false), + m_bConfirmRestartAsMaster(false), m_bShowDepth(false), m_bSocketThread(true), m_bFrameCompression(true), m_iCompressionLevel(2), m_pClientSocket(NULL), m_nFilterNeighbors(10), - m_fFilterThreshold(0.01f) + m_fFilterThreshold(0.01f), + m_bRestartingCamera(false), + m_bAutoExposureEnabled(true), // Which state the Auto Exposure should be set to + m_nExposureStep(-5) { pCapture = new AzureKinectCapture(); @@ -207,7 +211,7 @@ void LiveScanClient::UpdateFrame() if (m_bCaptureFrame) { uint64_t timeStamp = pCapture->GetTimeStamp(); - m_framesFileWriterReader.writeFrame(m_vLastFrameVertices, m_vLastFrameRGB, timeStamp); + m_framesFileWriterReader.writeFrame(m_vLastFrameVertices, m_vLastFrameRGB, timeStamp, pCapture->GetDeviceIndex()); m_bConfirmCaptured = true; m_bCaptureFrame = false; } @@ -277,7 +281,7 @@ LRESULT CALLBACK LiveScanClient::DlgProc(HWND hWnd, UINT message, WPARAM wParam, D2D1CreateFactory(D2D1_FACTORY_TYPE_SINGLE_THREADED, &m_pD2DFactory); // Get and initialize the default Kinect sensor as standalone - bool res = pCapture->Initialize(false, false, 0); + bool res = pCapture->Initialize(Standalone, 0); if (res) { calibration.LoadCalibration(pCapture->serialNumber); @@ -286,6 +290,8 @@ LRESULT CALLBACK LiveScanClient::DlgProc(HWND hWnd, UINT message, WPARAM wParam, m_pCameraSpaceCoordinates = new Point3f[pCapture->nDepthFrameWidth * pCapture->nDepthFrameHeight]; m_pColorInDepthSpace = new RGB[pCapture->nDepthFrameWidth * pCapture->nDepthFrameHeight]; + + pCapture->SetExposureState(true, 0); } else { @@ -455,30 +461,69 @@ void LiveScanClient::HandleSocket() //Determine if this device is a subordinate, master, or standalone int jackState = pCapture->GetSyncJackState(); + bool res = false; + switch (jackState) { case -1: currentTempSyncState = SUBORDINATE; + //Restart this device as Subordinate, with a unique syncOffset (send by the server) - pCapture->Close(); - pCapture->Initialize(false, true, syncOffset); + m_bRestartingCamera = true; + + + res = pCapture->Close(); + if (!res) { + SetStatusMessage(L"Subordinate device failed to close! Restart Application!", 10000, true); + return; + } + + res = pCapture->Initialize(Subordinate, syncOffset); + if (!res) { + SetStatusMessage(L"Subordinate device failed to reinitialize! Restart Application!", 10000, true); + return; + } //Confirm to the server, that we set this device as subordinate m_bConfirmTempSyncState = true; + m_bRestartingCamera = false; break; case 0: currentTempSyncState = MASTER; + //Only Close this device, as it needs to wait for all subordinates to start, before starting itself - pCapture->Close(); + m_bRestartingCamera = true; + + res = pCapture->Close(); + if (!res) { + SetStatusMessage(L"Master device failed to close! Restart Application!", 10000, true); + return; + } + m_bConfirmTempSyncState = true; break; case 1://Device is Standalone currentTempSyncState = STANDALONE; + //Restart this device as Standalone - pCapture->Close(); - pCapture->Initialize(false, false, 0); + m_bRestartingCamera = true; + + res = pCapture->Close(); + if (!res) { + SetStatusMessage(L"Capture device failed to close! Restart Application!", 10000, true); + return; + } + + res = pCapture->Initialize(Standalone, 0); + + if (!res) { + SetStatusMessage(L"Capture device failed to reinitialize! Restart Application!", 10000, true); + return; + } + m_bConfirmTempSyncState = true; + m_bRestartingCamera = false; break; default: break; @@ -488,16 +533,39 @@ void LiveScanClient::HandleSocket() //Sets this device as Standalone else if (received[i] == MSG_SET_TEMPSYNC_OFF) { currentTempSyncState = STANDALONE; - pCapture->Close(); - pCapture->Initialize(false, false, 0); + m_bRestartingCamera = true; + + bool res; + + res = pCapture->Close(); + if (!res) { + SetStatusMessage(L"Capture device failed to close! Restart Application!", 10000, true); + return; + } + + res = pCapture->Initialize(Standalone, 0); + + if (!res) { + SetStatusMessage(L"Capture device failed to reinitialize! Restart Application!", 10000, true); + return; + } + m_bConfirmTempSyncState = true; + m_bRestartingCamera = false; } //Got confirmation from the server that all subs have started, and we can now start the master else if (received[i] == MSG_START_MASTER) { if (currentTempSyncState == MASTER) { - pCapture->Initialize(true, false, 0); + bool res = pCapture->Initialize(Master, 0); + if (!res) { + SetStatusMessage(L"Master device failed to reinitialize! Restart Application!", 10000, true); + return; + } + + m_bConfirmRestartAsMaster = true; + m_bRestartingCamera = false; } } @@ -562,6 +630,14 @@ void LiveScanClient::HandleSocket() m_bFrameCompression = true; else m_bFrameCompression = false; + + m_bAutoExposureEnabled = (received[i] != 0); + i++; + + m_nExposureStep = *(int*)(received.c_str() + i); + i += sizeof(int); + + pCapture->SetExposureState(m_bAutoExposureEnabled, m_nExposureStep); //so that we do not lose the next character in the stream i--; @@ -615,6 +691,18 @@ void LiveScanClient::HandleSocket() { m_framesFileWriterReader.closeFileIfOpened(); } + + else if (received[i] == MSG_REQUEST_SYNC_JACK_STATE) + { + int size = 3; + char* buffer = new char[size]; + buffer[0] = MSG_SYNC_JACK_STATE; + + buffer[1] = pCapture->GetSyncJackState() + 1; //Gets the current Sync Jack State and adds + 1, as we can't send a negative value + + m_pClientSocket->SendBytes(buffer, size); + m_bConfirmTempSyncState = false; + } } if (m_bConfirmCaptured) @@ -642,6 +730,14 @@ void LiveScanClient::HandleSocket() m_bConfirmTempSyncState = false; } + //Send validation to the server that the Master camera has started recording again + if (m_bConfirmRestartAsMaster) + { + byteToSend = MSG_CONFIRM_MASTER_RESTART; + m_pClientSocket->SendBytes(&byteToSend, 1); + m_bConfirmRestartAsMaster = false; + } + if (m_bConfirmCalibrated) {