From 197f13fa5e21e55689ca203f6f6b991a0232d315 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 30 Mar 2022 23:47:53 -0400 Subject: [PATCH] Dockerfile and Github build These have a lot of cruft, but it should work. I'll switch over to using this from the python code in the Rust monorepo --- .dockerignore | 12 ++ .github/workflows/docker-publish.yml | 65 +++++++++++ Dockerfile | 164 +++++++++++++++++++++++++++ 3 files changed, 241 insertions(+) create mode 100644 .dockerignore create mode 100644 .github/workflows/docker-publish.yml create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e29cebd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +*secret*sh +*~ +.env +.env-secrets +.git +.idea/ +__pycache__/ +tts/models/Wav2Lip/python/ +tts/models/tacotron2/python/ +secrets.toml +secrets.yaml +target/ diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..0e8233d --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,65 @@ +# Adapted from +# https://github.com/actions/starter-workflows/ci/docker-publish.yml +# If using base images, don't forget to add READ access, eg. from a package settings page like: +# https://github.com/orgs/storytold/packages/container/docker-base-images-rust-ssl/settings/actions_access +name: Publish Docker Container to GHCR +on: + push: + branches: + # NB: Default-branch doesn't work, despite Github's documentation. + #- $default-branch + - master +env: + IMAGE_NAME: tts-core + +jobs: + # Push image to GitHub Packages. + # See also https://docs.docker.com/docker-hub/builds/ + push: + runs-on: ubuntu-latest + if: github.event_name == 'push' + permissions: + packages: write + contents: read + + steps: + - uses: actions/checkout@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + #username: ${{ github.repository_owner }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build image + run: docker build . --file Dockerfile --tag $IMAGE_NAME --label "runnumber=${GITHUB_RUN_ID}" + + - name: Push image to GitHub Container Registry + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME + + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `latest` tag convention + [ "$VERSION" == "$default-branch" ] && VERSION=latest + + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION + + SHORT_SHA=$(echo ${GITHUB_SHA} | cut -c1-12) + + docker tag $IMAGE_NAME $IMAGE_ID:$SHORT_SHA + docker push $IMAGE_ID:$SHORT_SHA + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..24ff5b5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,164 @@ +# ==================== Python Build Base ==================== + +FROM ghcr.io/storytold/docker-base-images-nvidia-cuda-experimental:080f96bc7087 as pybuild-base + +## Authenticating with GHCR locally: +## https://docs.github.com/en/packages/working-with-a-github-packages-registry/migrating-to-the-container-registry-from-the-docker-registry +## ``` +## export CR_PAT=YOUR_TOKEN +## echo $CR_PAT | docker login ghcr.io -u USERNAME --password-stdin +## ``` +# +##FROM nvidia/cuda:10.1-devel-ubuntu18.04 as pybuild-base +#FROM nvidia/cuda:11.3.0-devel-ubuntu20.04 as pybuild-base +# +## tzdata install hangs without presetting the time zone +#ENV TZ=UTC +#RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +# +## NB(2021-05-30): This is the image that Tacotron 2's Dockerfile specifies. +## I have not tried this, but it might be necessary. +##FROM pytorch/pytorch:nightly-devel-cuda10.0-cudnn7 +##ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} +# +## NB: https://github.com/NVIDIA/nvidia-docker/issues/864#issuecomment-439848887 +## NB: We do not install ffmpeg, since the version is 3.8.* series and we need 4.2.* +## +## NB(2021-05-30): We need to install `cython3` for building modern Tacotron 2 +## dependencies (ie. protoc) on old Ubuntu +## RUN apt-get update \ +## && apt-get install -y \ +#RUN apt-get update +##RUN apt-cache search python3.8 +# +## NB: Here we install nvidia drivers +## NB: Avoid keyboard prompt +#RUN DEBIAN_FRONTEND=noninteractive apt-get install -y nvidia-384 --no-install-recommends +# +## ffmpeg version 4.2.4-1ubuntu0.1 Copyright (c) 2000-2020 the FFmpeg developers +##RUN apt-get install -y ffmpeg +##RUN ffmpeg -version +##RUN sleep 100 +# +## Notes on packages: +## * software-properties-common - needed for `apt-add-repository` +## * pciutils - needed for `lspci | grep -i nvidia` +#RUN apt-get install -y \ +# build-essential \ +# curl \ +# cython3 \ +# ffmpeg \ +# g++-7 \ +# gcc-7 \ +# git \ +# htop \ +# imagemagick \ +# libc++-7-dev \ +# libffi-dev \ +# libgcc-7-dev \ +# libmagic1 \ +# libsm6 \ +# libsndfile1 \ +# libssl-dev \ +# libxext6 \ +# libxrender-dev \ +# netcat \ +# pciutils \ +# python-dev \ +# python3-pip \ +# python3.8 \ +# python3.8-dev \ +# python3.8-venv\ +# redis-tools \ +# silversearcher-ag \ +# software-properties-common \ +# sox \ +# tmux \ +# vim \ +# wget +# +# +## Wav2Lip needs Python3.6 +## NB: we install python3-virtualenv as `python3.6 -m venv` won't work anymore +#RUN add-apt-repository ppa:deadsnakes/ppa +#RUN apt-get update +#RUN apt-get install -y python3.6 +#RUN apt-get install -y python3-virtualenv +#RUN apt-get install -y python3.6-venv +#RUN apt-get install -y python3.6-dev +##RUN apt-get install python3-virtualenv +# +# +#RUN python3.6 --version +#RUN python3.8 --version + +# We need ffmpeg version >= 4.2 (https://superuser.com/a/579110) +# http://ubuntuhandbook.org/index.php/2020/06/install-ffmpeg-4-3-via-ppa-ubuntu-18-04-16-04/ + +#RUN apt-get install -y software-properties-common \ +# && add-apt-repository ppa:jonathonf/ffmpeg-4 \ +# && apt-get update \ +# && apt-get install -y ffmpeg +# + +# The downloader needs youtube-dl as a python library +# NB: No longer maintained: https://news.ycombinator.com/item?id=28319624 +#RUN pip3 install --upgrade youtube-dl + +# Used by the downloader script. +# And it looks like youtube-dl is dead... +# Per the README, it looks like pip will always pull the latest version. +# https://github.com/yt-dlp/yt-dlp#update +RUN python3 -m pip install --upgrade yt-dlp + +# ==================== Tacotron, Waveglow, HifiGan, etc. ==================== + +FROM pybuild-base as pybuild-requirements + +WORKDIR / + +COPY ./tts ./models/tts +WORKDIR models/tts + +# NB: Not sure if we need Python3.6 for Tacotron2, but Python3.8 gave me trouble. +# NB: Setuptools fix: https://github.com/tensorflow/tensorflow/issues/34302#issuecomment-554450289 +RUN python3.6 -m venv python +RUN . python/bin/activate \ + && pip install --upgrade pip setuptools \ + && pip install -r requirements-tacotron-python36.txt \ + && deactivate + + + +# Final image +# FROM ubuntu:xenial +FROM pybuild-requirements as final +WORKDIR / + +#COPY scripts/ ./scripts + +# See: https://github.com/opencontainers/image-spec/blob/master/annotations.md +LABEL org.opencontainers.image.authors='bt@brand.io, echelon@gmail.com' +LABEL org.opencontainers.image.url='https://github.com/storytold/tts-core' +LABEL org.opencontainers.image.documentation='https://github.com/storytold/tts-core' +LABEL org.opencontainers.image.source='https://github.com/storytold/tts-core' + +# NB: Comment this out for non-debug images +# TODO: Figure out how this is done elsewhere with just the static binaries +# Others: `mysql-client` +RUN apt-get update \ + && apt-get install -y \ + curl \ + python3-pip \ + python3 \ + wget + +RUN python3 --version +#RUN python3.7 -m venv python +#RUN . python/bin/activate \ +# && pip install --upgrade pip \ +# && pip install -r requirements.txt + +# TODO: Use venv to do this instead. +RUN pip3 install gdown +