{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#<----- Click this play button to setup MocapNET v4!\n", "\n", "#This is the Total Capture version that handles body, hands, gaze, face!\n", "#It also has been rewritten from scratch in Python for your convenience.\n", "#If you deploy this on your PC, run : jupyter notebook mocapnet4.ipynb\n", "# and remove --collab from the setup.sh invocation 2 lines below..!\n", "import os\n", "if (os.path.isfile(\"MocapNET.py\")):\n", " !git pull\n", " exit\n", "!git clone -b mnet4 https://github.com/FORTH-ModelBasedTracker/MocapNET.git\n", "!MocapNET/src/python/mnet4/setup.sh --collab\n", "os.chdir(\"MocapNET/src/python/mnet4\")\n", "print(\"MocapNET setup is finished, you can run the next cell now..\")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#<----- Click to get a video from the internet and then track it ( e.g. shuffle.webm )\n", "\n", "#You can also upload your own by clicking the Files icon and using the menu on the left\n", "#You need to put files in the /content/MocapNET/src/python/mnet4/ directory\n", "\n", "#Make sure we are at the correct directory\n", "import os\n", "if (not os.path.isfile(\"mediapipeHolisticWebcamMocapNET.py\")):\n", " os.chdir(\"MocapNET/src/python/mnet4\")\n", "\n", "!wget http://ammar.gr/mocapnet/shuffle.webm -O shuffle.webm\n", "\n", "#Analyze the file shuffle.web through MediaPipe 2D + MocapNET 3D Pose Estimation!\n", "!(python3 -m mediapipeHolisticWebcamMocapNET --from shuffle.webm --ik 0.001 99 99 --smooth 60 10 --all --save --plot --headless 2> /dev/null)\n", "print(\"MocapNET video input processing finished, you can run the next cells now to see the results..\")\n", "\n", "#Outputs are :\n", "#livelastRun3DHiRes.mp4 | A video showing the regressed output overlayed on RGB\n", "#livelastPlot3DHiRes.mp4| A video plot of the retrieved BVH degrees of freedom\n", "#out.bvh | the extracted BVH file \n", "#2d_out.csv | Input 2D joints used by MocapNET as a CSV file\n", "#3d_out.csv | Output 3D joints produced by MocapNET as a CSV file\n", "#bvh_out.csv | Output BVH angles produced by MocapNET as a CSV file\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#<----- Click to run a sign-language dataset without mediapipe from a dumped Openpose/JSON/CSV source\n", "\n", "#Make sure we are at the correct directory\n", "import os\n", "if (not os.path.isfile(\"csvNET.py\")):\n", " os.chdir(\"MocapNET/src/python/mnet4\")\n", "\n", "#Download a single sign (con0014) from SIGNUM\n", "!wget http://ammar.gr/datasets/signumtest.zip -O signumtest.zip\n", "!unzip -o signumtest.zip\n", "\n", "#Analyze the file con0014/2dJoints_v1.4.csv through MediaPipe 2D + MocapNET 3D Pose Estimation!\n", "!(python3 -m csvNET --from con0014/2dJoints_v1.4.csv --ik 0.001 99 99 --upperbody --reye --mouth --hands --save --plot --headless 2> /dev/null)\n", "print(\"MocapNET video input processing finished, you can run the next cells now to see the results..\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#<----- Click to see the generated BVH angle plot of the last MocapNET run!\n", "from IPython.display import Video\n", "import os\n", "if (not os.path.isfile(\"livelastPlot3DHiRes.mp4\")):\n", " print(\"Please execute the previous cells and wait for them to complete before seeing this visualization!\")\n", " exit\n", "\n", "Video(\"livelastPlot3DHiRes.mp4\",embed=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#<----- Click to see the generated visualization of the last MocapNET run!\n", "from IPython.display import Video\n", "import os\n", "if (not os.path.isfile(\"livelastRun3DHiRes.mp4\")):\n", " print(\"Please execute the previous cells and wait for them to complete before seeing this visualization!\")\n", " exit\n", "\n", "Video(\"livelastRun3DHiRes.mp4\",embed=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#<----- Click to download the generated BVH/3D files of the last MocapNET run!\n", "import os \n", "from google.colab import files\n", "if (not os.path.isfile(\"out.bvh\")) or (not os.path.isfile(\"2d_out.csv\")) or (not os.path.isfile(\"bvh_out.csv\")) or (not os.path.isfile(\"3d_out.csv\")):\n", " print(\"Please execute the previous cells and wait for them to complete before downloading output!\")\n", "else:\n", " print(\"Download Output Files!\")\n", " files.download(\"out.bvh\")\n", " files.download(\"2d_out.csv\")\n", " files.download(\"3d_out.csv\")\n", " files.download(\"bvh_out.csv\")\n", "#To render the 3D face get Blender from https://www.blender.org/\n", "#Download http://ammar.gr/mocapnet/mnet4/face.blend and open it using Blender\n", "#Download http://ammar.gr/mocapnet/mnet4/headerWithHeadAndOneMotion.bvh and open it using Blender at a 0.01 scale\n", "#Run The Loaded Python Script\n", "#Click on the armature and on the menu right click the orange rectangle\n", "#Select headerWithHeadAndOneMotion as Source BVH\n", "#Select newgirl as Target Obj\n", "#Select a directory as a target for generated dataset\n", "#Select the path to the regressed bvh_out.csv to load pre-generated dataset\n", "#Click Just Render CSV Dataset\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#<----- Click to attempt MocapNET live demo by streaming your camera to Google Colab\n", "#EXPERIMENTAL, performance is impacted because of low image throughput!\n", "\n", "import cv2\n", "import os\n", "from colabStream import init_camera,take_frame,show_frame\n", "\n", " \n", "from mediapipeHolisticWebcamMocapNET import MediaPipeHolistic\n", "from MocapNET import easyMocapNETConstructor\n", "from folderStream import resize_with_padding\n", "\n", "# Create instances of MediaPipeHolistic and MocapNET\n", "mp_holistic = MediaPipeHolistic(doMediapipeVisualization=False)\n", "mnet = easyMocapNETConstructor(\n", " engine = \"onnx\",\n", " doProfiling = False,\n", " multiThreaded = False,\n", " doHCDPostProcessing = True,\n", " hcdLearningRate = 0.001,\n", " hcdEpochs = 99,\n", " hcdIterations = 99,\n", " bvhScale = 1.0,\n", " doBody = True,\n", " doFace = False,\n", " doREye = True,\n", " doMouth = True,\n", " doHands = True,\n", " addNoise = 0.0\n", ")\n", "\n", "\n", "# init JavaScript code\n", "init_camera()\n", "\n", "while True:\n", " try:\n", " image = take_frame()\n", " \n", " image = resize_with_padding(image,1280,720)\n", "\n", " # Perform image processing with MediaPipeHolistic\n", " mocapNETInput, annotated_image = mp_holistic.convertImageToMocapNETInput(image)\n", "\n", " # Perform 3D joint prediction with MocapNET\n", " mocapNET3DOutput = mnet.predict3DJoints(mocapNETInput)\n", "\n", " # Visualize the results on the image\n", " #image_with_results = Image.fromarray(cv2.cvtColor(annotated_image, cv2.COLOR_BGR2RGB))\n", " \n", " from MocapNETVisualization import visualizeMocapNETEnsemble\n", " image,plotImage = visualizeMocapNETEnsemble(mnet,annotated_image,plotBVHChannels=False)\n", " \n", " show_frame(annotated_image) # it replace previous image\n", " \n", " except Exception as err:\n", " print('Exception:', err)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "#D/L and compile a different version of MocapNET and evaluate it against CMUBVH test dataset\n", "#Running this will take a while, but it is usefult to compare different trained MocapNET Models!\n", "import os\n", "\n", "#Download Test Set!\n", "if (not os.path.isfile(\"dataset/generated/bvh_body_all_test.csv\")):\n", " print(\"Downloading test set!\")\n", " os.system(\"wget http://ammar.gr/datasets/testBody.zip && unzip testBody.zip\")\n", "\n", "#Download a different MocapNET build\n", "!python3 -m getModelFromDatabase --get 319 #<- change number to try a different version\n", "\n", "#Do Evaluation\n", "!python3 -m evaluateMocapNET --config dataset/body_configuration.json --all body --skip 5 --engine onnx > lastEvaluationLog.txt\n", "!tail -n 30 lastEvaluationLog.txt " ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\"\"\"\n", "\n", "This library is provided under the FORTH license\n", "https://github.com/FORTH-ModelBasedTracker/MocapNET/blob/master/license.txt\n", "\n", "If you use this version of MocapNET for your research please consider citing : \n", "\n", "@inproceedings{Qammaz2023b,\n", " author = {Qammaz, Ammar and Argyros, Antonis},\n", " title = {A Unified Approach for Occlusion Tolerant 3D Facial Pose Capture and Gaze Estimation using MocapNETs},\n", " booktitle = {International Conference on Computer Vision Workshops (AMFG 2023 - ICCVW 2023), (to appear)},\n", " publisher = {IEEE},\n", " year = {2023},\n", " month = {October},\n", " address = {Paris, France},\n", " projects = {VMWARE,I.C.HUMANS},\n", " pdflink = {http://users.ics.forth.gr/ argyros/mypapers/2023_10_AMFG_Qammaz.pdf}\n", "}\n", "\n", "@inproceedings{Qammaz2021,\n", " author = {Qammaz, Ammar and Argyros, Antonis A},\n", " title = {Towards Holistic Real-time Human 3D Pose Estimation using MocapNETs},\n", " booktitle = {British Machine Vision Conference (BMVC 2021)},\n", " publisher = {BMVA},\n", " year = {2021},\n", " month = {November},\n", " projects = {I.C.HUMANS},\n", " videolink = {https://www.youtube.com/watch?v=aaLOSY_p6Zc}\n", "}\n", "\"\"\" " ] } ], "metadata": { "colab": { "provenance": [] }, "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 1 }