storyteller-ml/animation/MocapNET/mocapnet4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#<----- Click this play button to setup MocapNET v4!\n",
    "\n",
    "#This is the Total Capture version that handles body, hands, gaze, face!\n",
    "#It also has been rewritten from scratch in Python for your convenience.\n",
    "#If you deploy this on your PC, run : jupyter notebook mocapnet4.ipynb\n",
    "#   and remove --collab from the setup.sh invocation 2 lines below..!\n",
    "import os\n",
    "if (os.path.isfile(\"MocapNET.py\")):\n",
    "    !git pull\n",
    "    exit\n",
    "!git clone -b mnet4 https://github.com/FORTH-ModelBasedTracker/MocapNET.git\n",
    "!MocapNET/src/python/mnet4/setup.sh --collab\n",
    "os.chdir(\"MocapNET/src/python/mnet4\")\n",
    "print(\"MocapNET setup is finished, you can run the next cell now..\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#<----- Click to get a video from the internet and then track it ( e.g. shuffle.webm )\n",
    "\n",
    "#You can also upload your own by clicking the Files icon and using the menu on the left\n",
    "#You need to put files in the /content/MocapNET/src/python/mnet4/ directory\n",
    "\n",
    "#Make sure we are at the correct directory\n",
    "import os\n",
    "if (not os.path.isfile(\"mediapipeHolisticWebcamMocapNET.py\")):\n",
    "    os.chdir(\"MocapNET/src/python/mnet4\")\n",
    "\n",
    "!wget http://ammar.gr/mocapnet/shuffle.webm -O shuffle.webm\n",
    "\n",
    "#Analyze the file shuffle.web through MediaPipe 2D + MocapNET 3D Pose Estimation!\n",
    "!(python3 -m mediapipeHolisticWebcamMocapNET --from shuffle.webm --ik 0.001 99 99 --smooth 60 10 --all --save --plot --headless 2> /dev/null)\n",
    "print(\"MocapNET video input processing finished, you can run the next cells now to see the results..\")\n",
    "\n",
    "#Outputs are :\n",
    "#livelastRun3DHiRes.mp4 | A video showing the regressed output overlayed on RGB\n",
    "#livelastPlot3DHiRes.mp4| A video plot of the retrieved BVH degrees of freedom\n",
    "#out.bvh                | the extracted BVH file \n",
    "#2d_out.csv             | Input 2D joints used by MocapNET as a CSV file\n",
    "#3d_out.csv             | Output 3D joints produced by MocapNET as a CSV file\n",
    "#bvh_out.csv            | Output BVH angles produced by MocapNET as a CSV file\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#<----- Click to run a sign-language dataset without mediapipe from a dumped Openpose/JSON/CSV source\n",
    "\n",
    "#Make sure we are at the correct directory\n",
    "import os\n",
    "if (not os.path.isfile(\"csvNET.py\")):\n",
    "    os.chdir(\"MocapNET/src/python/mnet4\")\n",
    "\n",
    "#Download a single sign (con0014) from SIGNUM\n",
    "!wget http://ammar.gr/datasets/signumtest.zip -O signumtest.zip\n",
    "!unzip -o signumtest.zip\n",
    "\n",
    "#Analyze the file con0014/2dJoints_v1.4.csv through MediaPipe 2D + MocapNET 3D Pose Estimation!\n",
    "!(python3 -m csvNET --from con0014/2dJoints_v1.4.csv --ik 0.001 99 99 --upperbody --reye --mouth --hands --save --plot --headless 2> /dev/null)\n",
    "print(\"MocapNET video input processing finished, you can run the next cells now to see the results..\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#<----- Click to see the generated BVH angle plot of the last MocapNET run!\n",
    "from IPython.display import Video\n",
    "import os\n",
    "if (not os.path.isfile(\"livelastPlot3DHiRes.mp4\")):\n",
    "   print(\"Please execute the previous cells and wait for them to complete before seeing this visualization!\")\n",
    "   exit\n",
    "\n",
    "Video(\"livelastPlot3DHiRes.mp4\",embed=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#<----- Click to see the generated visualization of the last MocapNET run!\n",
    "from IPython.display import Video\n",
    "import os\n",
    "if (not os.path.isfile(\"livelastRun3DHiRes.mp4\")):\n",
    "   print(\"Please execute the previous cells and wait for them to complete before seeing this visualization!\")\n",
    "   exit\n",
    "\n",
    "Video(\"livelastRun3DHiRes.mp4\",embed=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#<----- Click to download the generated BVH/3D files of the last MocapNET run!\n",
    "import os \n",
    "from google.colab import files\n",
    "if (not os.path.isfile(\"out.bvh\")) or (not os.path.isfile(\"2d_out.csv\")) or (not os.path.isfile(\"bvh_out.csv\")) or (not os.path.isfile(\"3d_out.csv\")):\n",
    "    print(\"Please execute the previous cells and wait for them to complete before downloading output!\")\n",
    "else:\n",
    "    print(\"Download Output Files!\")\n",
    "    files.download(\"out.bvh\")\n",
    "    files.download(\"2d_out.csv\")\n",
    "    files.download(\"3d_out.csv\")\n",
    "    files.download(\"bvh_out.csv\")\n",
    "#To render the 3D face get Blender from https://www.blender.org/\n",
    "#Download http://ammar.gr/mocapnet/mnet4/face.blend and open it using Blender\n",
    "#Download http://ammar.gr/mocapnet/mnet4/headerWithHeadAndOneMotion.bvh and open it using Blender at a 0.01 scale\n",
    "#Run The Loaded Python Script\n",
    "#Click on the armature and on the menu right click the orange rectangle\n",
    "#Select headerWithHeadAndOneMotion as Source BVH\n",
    "#Select newgirl as Target Obj\n",
    "#Select a directory as a target for generated dataset\n",
    "#Select the path to the regressed bvh_out.csv to load pre-generated dataset\n",
    "#Click Just Render CSV Dataset\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#<----- Click to attempt MocapNET live demo by streaming your camera to Google Colab\n",
    "#EXPERIMENTAL, performance is impacted because of low image throughput!\n",
    "\n",
    "import cv2\n",
    "import os\n",
    "from colabStream import init_camera,take_frame,show_frame\n",
    "\n",
    "  \n",
    "from mediapipeHolisticWebcamMocapNET import MediaPipeHolistic\n",
    "from MocapNET import easyMocapNETConstructor\n",
    "from folderStream import resize_with_padding\n",
    "\n",
    "# Create instances of MediaPipeHolistic and MocapNET\n",
    "mp_holistic = MediaPipeHolistic(doMediapipeVisualization=False)\n",
    "mnet = easyMocapNETConstructor(\n",
    "    engine              = \"onnx\",\n",
    "    doProfiling         = False,\n",
    "    multiThreaded       = False,\n",
    "    doHCDPostProcessing = True,\n",
    "    hcdLearningRate     = 0.001,\n",
    "    hcdEpochs           = 99,\n",
    "    hcdIterations       = 99,\n",
    "    bvhScale            = 1.0,\n",
    "    doBody              = True,\n",
    "    doFace              = False,\n",
    "    doREye              = True,\n",
    "    doMouth             = True,\n",
    "    doHands             = True,\n",
    "    addNoise            = 0.0\n",
    ")\n",
    "\n",
    "\n",
    "# init JavaScript code\n",
    "init_camera()\n",
    "\n",
    "while True:\n",
    "    try:\n",
    "        image = take_frame()\n",
    " \n",
    "        image = resize_with_padding(image,1280,720)\n",
    "\n",
    "        # Perform image processing with MediaPipeHolistic\n",
    "        mocapNETInput, annotated_image = mp_holistic.convertImageToMocapNETInput(image)\n",
    "\n",
    "        # Perform 3D joint prediction with MocapNET\n",
    "        mocapNET3DOutput = mnet.predict3DJoints(mocapNETInput)\n",
    "\n",
    "        # Visualize the results on the image\n",
    "        #image_with_results = Image.fromarray(cv2.cvtColor(annotated_image, cv2.COLOR_BGR2RGB))\n",
    " \n",
    "        from MocapNETVisualization import visualizeMocapNETEnsemble\n",
    "        image,plotImage = visualizeMocapNETEnsemble(mnet,annotated_image,plotBVHChannels=False)\n",
    "    \n",
    "        show_frame(annotated_image)  # it replace previous image\n",
    "        \n",
    "    except Exception as err:\n",
    "        print('Exception:', err)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#D/L and compile a different version of MocapNET and evaluate it against CMUBVH test dataset\n",
    "#Running this will take a while, but it is usefult to compare different trained MocapNET Models!\n",
    "import os\n",
    "\n",
    "#Download Test Set!\n",
    "if (not os.path.isfile(\"dataset/generated/bvh_body_all_test.csv\")):\n",
    "   print(\"Downloading test set!\")\n",
    "   os.system(\"wget http://ammar.gr/datasets/testBody.zip && unzip testBody.zip\")\n",
    "\n",
    "#Download a different MocapNET build\n",
    "!python3 -m getModelFromDatabase --get 319 #<- change number to try a different version\n",
    "\n",
    "#Do Evaluation\n",
    "!python3 -m evaluateMocapNET --config dataset/body_configuration.json --all body --skip 5 --engine onnx > lastEvaluationLog.txt\n",
    "!tail -n 30 lastEvaluationLog.txt "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe width=\"560\" height=\"315\" src=\"https://youtube.com/embed/ooLRUS5j4AI\"\n",
       "</iframe>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%HTML\n",
    "<!-- Activate this block to see a youtube video about rendering MocapNET results with Blender -->\n",
    "<iframe width=\"560\" height=\"315\" src=\"https://youtube.com/embed/ooLRUS5j4AI\"></iframe>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "\n",
    "This library is provided under the FORTH license\n",
    "https://github.com/FORTH-ModelBasedTracker/MocapNET/blob/master/license.txt\n",
    "\n",
    "If you use this version of MocapNET for your research please consider citing : \n",
    "\n",
    "@inproceedings{Qammaz2023b,\n",
    "  author = {Qammaz, Ammar and Argyros, Antonis},\n",
    "  title = {A Unified Approach for Occlusion Tolerant 3D Facial Pose Capture and Gaze Estimation using MocapNETs},\n",
    "  booktitle = {International Conference on Computer Vision Workshops (AMFG 2023 - ICCVW 2023), (to appear)},\n",
    "  publisher = {IEEE},\n",
    "  year = {2023},\n",
    "  month = {October},\n",
    "  address = {Paris, France},\n",
    "  projects =  {VMWARE,I.C.HUMANS},\n",
    "  pdflink = {http://users.ics.forth.gr/ argyros/mypapers/2023_10_AMFG_Qammaz.pdf}\n",
    "}\n",
    "\n",
    "@inproceedings{Qammaz2021,\n",
    "  author = {Qammaz, Ammar and Argyros, Antonis A},\n",
    "  title = {Towards Holistic Real-time Human 3D Pose Estimation using MocapNETs},\n",
    "  booktitle = {British Machine Vision Conference (BMVC 2021)},\n",
    "  publisher = {BMVA},\n",
    "  year = {2021},\n",
    "  month = {November},\n",
    "  projects =  {I.C.HUMANS},\n",
    "  videolink = {https://www.youtube.com/watch?v=aaLOSY_p6Zc}\n",
    "}\n",
    "\"\"\" "
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}