diff --git a/.gitignore b/.gitignore index 690df8f..eecf2f0 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ temp/ venv/ env/ workflow/ +deepfake/ gfpgan/ models/inswapper_128.onnx models/GFPGANv1.4.pth diff --git a/README.md b/README.md index f108125..fd103a4 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ![demo-gif](demo.gif) - ## Disclaimer + This software is meant to be a productive contribution to the rapidly growing AI-generated media industry. It will help artists with tasks such as animating a custom character or using the character as a model for clothing etc. The developers of this software are aware of its possible unethical applications and are committed to take preventative measures against them. It has a built-in check which prevents the program from working on inappropriate media including but not limited to nudity, graphic content, sensitive material such as war footage etc. We will continue to develop this project in the positive direction while adhering to law and ethics. This project may be shut down or include watermarks on the output if requested by law. @@ -10,38 +10,43 @@ Users of this software are expected to use this software responsibly while abidi ## How do I install it? - ### Basic: It is more likely to work on your computer but it will also be very slow. You can follow instructions for the basic install (This usually runs via **CPU**) + #### 1.Setup your platform -- python (3.10 recommended) -- pip -- git -- [ffmpeg](https://www.youtube.com/watch?v=OlNWCpFdVMA) -- [visual studio 2022 runtimes (windows)](https://visualstudio.microsoft.com/visual-cpp-build-tools/) + +- python (3.10 recommended) +- pip +- git +- [ffmpeg](https://www.youtube.com/watch?v=OlNWCpFdVMA) +- [visual studio 2022 runtimes (windows)](https://visualstudio.microsoft.com/visual-cpp-build-tools/) + #### 2. Clone Repository + https://github.com/hacksider/Deep-Live-Cam.git #### 3. Download Models - 1. [GFPGANv1.4](https://huggingface.co/hacksider/deep-live-cam/resolve/main/GFPGANv1.4.pth) - 2. [inswapper_128_fp16.onnx](https://huggingface.co/hacksider/deep-live-cam/resolve/main/inswapper_128_fp16.onnx) +1. [GFPGANv1.4](https://huggingface.co/hacksider/deep-live-cam/resolve/main/GFPGANv1.4.pth) +2. [inswapper_128_fp16.onnx](https://huggingface.co/hacksider/deep-live-cam/resolve/main/inswapper_128_fp16.onnx) Then put those 2 files on the "**models**" folder #### 4. Install dependency -We highly recommend to work with a `venv` to avoid issues. + +We highly recommend to work with a `venv` to avoid issues. + ``` pip install -r requirements.txt ``` + ##### DONE!!! If you dont have any GPU, You should be able to run roop using `python run.py` command. Keep in mind that while running the program for first time, it will download some models which can take time depending on your network connection. -### *Proceed if you want to use GPU Acceleration -### CUDA Execution Provider (Nvidia)* +### \*Proceed if you want to use GPU Acceleration -1. Install [CUDA Toolkit 11.8](https://developer.nvidia.com/cuda-11-8-0-download-archive) - +### CUDA Execution Provider (Nvidia)\* + +1. Install [CUDA Toolkit 11.8](https://developer.nvidia.com/cuda-11-8-0-download-archive) 2. Install dependencies: - ``` pip uninstall onnxruntime onnxruntime-gpu @@ -124,6 +129,7 @@ python run.py --execution-provider openvino ``` ## How do I use it? + > Note: When you run this program for the first time, it will download some models ~300MB in size. Executing `python run.py` command will launch this window: @@ -132,16 +138,39 @@ Executing `python run.py` command will launch this window: Choose a face (image with desired face) and the target image/video (image/video in which you want to replace the face) and click on `Start`. Open file explorer and navigate to the directory you select your output to be in. You will find a directory named `` where you can see the frames being swapped in realtime. Once the processing is done, it will create the output file. That's it. ## For the webcam mode + Just follow the clicks on the screenshot + 1. Select a face 2. Click live 3. Wait for a few seconds (it takes a longer time, usually 10 to 30 seconds before the preview shows up) ![demo-gif](demo.gif) -Just use your favorite screencapture to stream like OBS -> Note: In case you want to change your face, just select another picture, the preview mode will then restart (so just wait a bit). +## TO USE WITH GOOGLE COLAB +1. Upload the colab file in /google-colab/DeepLive_Google_Colab.ipynb to your google colab +2. Follow the instructions to run the colab file. + Note: For TCP Tunneling you can use either Ngrok or FRP. + i. For Ngrok you would need an api key and payment details added to use the tcp connection + ii. For FRP its free but you need to host the FRPS server on a VPS and replace '194.113.64.71' with your vps address +3. Follow the image instruction to input the tcp address appropriately + On Colab + + ![gui-demo](colab_tcp_tunnel.png) + + On your Machine + + ![gui-demo](colab_instruction.png) + colab_instruction.png + +4. For live Mode add the source image, enable Remote Processor,Fill the tcp correctly and click on live + +5. For Image swap add the source and target image, enable Remote Processor, click on preview + +Just use your favorite screencapture to stream like OBS + +> Note: In case you want to change your face, just select another picture, the preview mode will then restart (so just wait a bit). Additional command line arguments are given below. To learn out what they do, check [this guide](https://github.com/s0md3v/roop/wiki/Advanced-Options). @@ -167,6 +196,7 @@ options: Looking for a CLI mode? Using the -s/--source argument will make the run program in cli mode. ## Want the Next Update Now? + If you want the latest and greatest build, or want to see some new great features, go to our [experimental branch](https://github.com/hacksider/Deep-Live-Cam/tree/experimental) and experience what the contributors have given. ## Credits diff --git a/colab_demo.mov b/colab_demo.mov new file mode 100644 index 0000000..71e04bb Binary files /dev/null and b/colab_demo.mov differ diff --git a/google-colab/DeepLive_Google_Colab.ipynb b/google-colab/DeepLive_Google_Colab.ipynb new file mode 100644 index 0000000..c2d25a6 --- /dev/null +++ b/google-colab/DeepLive_Google_Colab.ipynb @@ -0,0 +1,1502 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "U3Fc3OXuDeqP" + }, + "source": [ + "# **Install InsightFace and Dependencies:**\n", + "\n", + " Run the following code to install the required packages.\n", + "\n", + " Note: This installation varies and depends on your cuda and cudnn version. Incase of face swapping issue refer to the link in the debug session to know the version to install." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "id": "D-UgHjSFBE9d" + }, + "outputs": [], + "source": [ + "#!pip install onnx==1.16.0\n", + "!pip install onnxruntime-gpu==1.18.0 --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/\n", + "!pip install insightface #==0.7.3\n", + "#!pip install onnxruntime==1.18.0\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Q81Sf-CFarjq" + }, + "outputs": [], + "source": [ + "!apt-get install -y ffmpeg" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GYFbuuCdvTaO" + }, + "source": [ + "### DeepFakeLive\n", + "Follow the steps below to create the necessery directories for the project" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "_MDoAtpyvV_l" + }, + "outputs": [], + "source": [ + "!cd /content\n", + "!rm -rf *" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "RzPl9sPVgSoP" + }, + "outputs": [], + "source": [ + "!mkdir deepfakecollab" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "lJJbUDokN0bF", + "outputId": "a2ef8b6e-97a8-4ef2-f986-e34e6eb1252e" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/content/deepfakecollab\n" + ] + } + ], + "source": [ + "cd deepfakecollab" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "ZUzAF6L2N6__" + }, + "outputs": [], + "source": [ + "!mkdir Scripts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ziSLHxab_j0h" + }, + "outputs": [], + "source": [ + "!ffmpeg -version" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0rTCd2R1uw7A" + }, + "source": [ + "### [Optional] FRP\n", + "\n", + "Follow the steps below to setup FRP. You will also need to host the FRPS on your VPS (free).\n", + "\n", + "Note: You cannot run both FRP and ngrok together. Its ether FRP and ngrok" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ckp_dM52N9K3" + }, + "outputs": [], + "source": [ + "!touch Scripts/get_frs.sh\n", + "\n", + "getfrs = \"\"\"#!/usr/bin/env bash\n", + "\n", + "# Check if frpc is installed\n", + "command -v frpc >/dev/null 2>&1\n", + "if [[ $? -ne 0 ]]; then\n", + " echo \"frpc is not found, installing...\"\n", + " wget -q -nc https://github.com/fatedier/frp/releases/download/v0.59.0/frp_0.59.0_linux_amd64.tar.gz\n", + " tar -xzf frp_0.59.0_linux_amd64.tar.gz\n", + " echo \"Done!\"\n", + "fi\"\"\"\n", + "with open('Scripts/get_frs.sh', 'w') as f:\n", + " f.write(getfrs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "w21kxVRmODFZ" + }, + "outputs": [], + "source": [ + "!touch Scripts/open_tunnel_frs.sh\n", + "\n", + "getfrs = \"\"\"#!/usr/bin/env bash\n", + "\n", + "cmd=\"frp_0.59.0_linux_amd64/frpc -c frp_0.59.0_linux_amd64/frpc.toml\"\n", + "\n", + "kill -9 $(ps aux | grep $cmd | awk '{print $2}') 2> /dev/null\n", + "\n", + "echo Opening tunnel\n", + "$cmd\"\"\"\n", + "with open('Scripts/open_tunnel_frs.sh', 'w') as f:\n", + " f.write(getfrs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "tmOY654Mia79" + }, + "outputs": [], + "source": [ + "!chmod +x Scripts/get_frs.sh\n", + "!chmod +x Scripts/open_tunnel_frs.sh" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Br-P5pkOwFg1", + "outputId": "663dae11-27c6-4329-a768-58faa1dd14e9" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "frpc is not found, installing...\n", + "Done!\n" + ] + } + ], + "source": [ + "!Scripts/get_frs.sh" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9n-qESinvM22" + }, + "source": [ + "### [Optional] Ngrok\n", + "\n", + "Follow to Setup Ngrok.\n", + "\n", + "Note: You need an API key from Ngrok to use tcp for free you would need to add a billing details to their platform" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "id": "S9U6x85DvTTV" + }, + "outputs": [], + "source": [ + "!touch Scripts/get_ngrok.sh\n", + "\n", + "getfrs = \"\"\"#!/usr/bin/env bash\n", + "\n", + "# Check if frpc is installed\n", + "command -v frpc >/dev/null 2>&1\n", + "if [[ $? -ne 0 ]]; then\n", + " echo \"ngrok is not found, installing...\"\n", + " wget -q -nc https://bin.equinox.io/c/bNyj1mQVY4c/ngrok-v3-stable-linux-amd64.tgz\n", + " tar -xzf ngrok-v3-stable-linux-amd64.tgz\n", + " echo \"Done!\"\n", + "fi\"\"\"\n", + "with open('Scripts/get_ngrok.sh', 'w') as f:\n", + " f.write(getfrs)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "ZYKKKRZSwA82" + }, + "outputs": [], + "source": [ + "!touch Scripts/open_tunnel_ngrok.sh\n", + "\n", + "getfrs = \"\"\"#!/usr/bin/env bash\n", + "\n", + "cmd=\"./ngrok start --all --config ngrok.conf\"\n", + "\n", + "kill -9 $(ps aux | grep $cmd | awk '{print $2}') 2> /dev/null\n", + "\n", + "echo Opening tunnel\n", + "$cmd\"\"\"\n", + "with open('Scripts/open_tunnel_ngrok.sh', 'w') as f:\n", + " f.write(getfrs)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "id": "A2FQrmDQwW__" + }, + "outputs": [], + "source": [ + "!chmod +x Scripts/get_ngrok.sh\n", + "!chmod +x Scripts/open_tunnel_ngrok.sh" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "WoNs1gSiweBm", + "outputId": "9e24a0a6-b50b-49f6-a928-2c004917927e" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ngrok is not found, installing...\n", + "Done!\n" + ] + } + ], + "source": [ + "!Scripts/get_ngrok.sh" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "id": "uCPALISZwo3K" + }, + "outputs": [], + "source": [ + "# Paste your authtoken here in quotes\n", + "authtoken = \"\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uVpDmuTmwtO9" + }, + "source": [ + "Set your region\n", + "\n", + "Code | Region\n", + "--- | ---\n", + "us | United States\n", + "eu | Europe\n", + "ap | Asia/Pacific\n", + "au | Australia\n", + "sa | South America\n", + "jp | Japan\n", + "in | India" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "id": "MM-UUANCwwqo" + }, + "outputs": [], + "source": [ + "# Set your region here in quotes\n", + "region = \"eu\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "JmNGrasvwnVX" + }, + "source": [ + "### Create Model Folder\n", + "\n", + "Create and download into the model folder" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "id": "rYAgrA75wv4U" + }, + "outputs": [], + "source": [ + "!mkdir -p Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "EU1ZXGlGw8Rb" + }, + "outputs": [], + "source": [ + "!wget https://github.com/facefusion/facefusion-assets/releases/download/models/inswapper_128_fp16.onnx -P /content/deepfakecollab/Model\n", + "!wget https://huggingface.co/hacksider/deep-live-cam/resolve/main/GFPGANv1.4.pth -P /content/deepfakecollab/Model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sNHGT2FjjZiK" + }, + "source": [ + "### [Ignore] DeBuggin (Run only when its necessery)\n", + "\n", + "Debugging to ensure cuda was used. if not check the version of the cudnn, edit the install of the onnxruntime package to install the right version. https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#requirements" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ZQr-gWl6Ibw7", + "outputId": "ef468d22-a4c5-4f73-932b-dea535cc25ed" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "#define CUDNN_MAJOR 8\n", + "#define CUDNN_MINOR 9\n", + "#define CUDNN_PATCHLEVEL 6\n", + "--\n", + "#define CUDNN_VERSION (CUDNN_MAJOR * 1000 + CUDNN_MINOR * 100 + CUDNN_PATCHLEVEL)\n", + "\n", + "/* cannot use constexpr here since this is a C-only file */\n" + ] + } + ], + "source": [ + "!cat /usr/include/cudnn_version.h | grep CUDNN_MAJOR -A 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "RZ9r-s_9Kdha", + "outputId": "b8778f0e-f10a-4725-cac7-67a435992100" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.18.0\n" + ] + } + ], + "source": [ + "import onnxruntime as rt\n", + "print(rt.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Stwb4OHv_hs0", + "outputId": "41d0963a-31df-4fea-f80b-cbebef56d07c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Active providers: ['CUDAExecutionProvider', 'CPUExecutionProvider']\n" + ] + } + ], + "source": [ + "# Ensure the GPU providers are set explicitly\n", + "ort_session = rt.InferenceSession(\n", + " \"/content/deepfakecollab/Model/inswapper_128_fp16.onnx\",\n", + " providers=[\"CUDAExecutionProvider\", \"CPUExecutionProvider\"]\n", + ")\n", + "\n", + "# Verify the active provider again\n", + "print(\"Active providers:\", ort_session.get_providers())\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DTbmyO-CK-8b" + }, + "source": [ + "### Create the Colab Server\n", + "\n", + "In your Google Colab notebook, set up a TCP server that will receive frames, process them using the FACE_SWAPPER model, and send back the results." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "id": "V_Luv36jjcmH" + }, + "outputs": [], + "source": [ + "import socket\n", + "import cv2\n", + "import numpy as np\n", + "import insightface\n", + "import threading\n", + "import torch\n", + "import onnxruntime\n", + "from typing import Any\n", + "from insightface.app.common import Face\n", + "import matplotlib.pyplot as plt\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "P4xE5bpeosP_", + "outputId": "7562a24e-ca5f-491e-884e-ce43b2ca0325" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['CUDAExecutionProvider']\n" + ] + } + ], + "source": [ + "# Check if CUDA is available\n", + "if 'CUDAExecutionProvider' in onnxruntime.get_available_providers():\n", + " providers = ['CUDAExecutionProvider']\n", + "elif 'TensorrtExecutionProvider' in onnxruntime.get_available_providers():\n", + " providers = ['TensorrtExecutionProvider']\n", + "else:\n", + " providers = ['CPUExecutionProvider']\n", + "print(providers)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "0OFrqUjgz7zb" + }, + "outputs": [], + "source": [ + "\n", + "FACE_SWAPPER = None\n", + "FACE_ANALYSER = insightface.app.FaceAnalysis(name='buffalo_l', providers=providers)\n", + "FACE_ANALYSER.prepare(ctx_id=0, det_size=(640, 640))\n", + "THREAD_LOCK = threading.Lock()\n", + "Frame = np.ndarray[Any, Any]" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "id": "ZbPSlDq_zJLj" + }, + "outputs": [], + "source": [ + "def get_face_swapper() -> Any:\n", + " global FACE_SWAPPER\n", + "\n", + " with THREAD_LOCK:\n", + " if FACE_SWAPPER is None:\n", + " model_path = \"/content/deepfakecollab/Model/inswapper_128_fp16.onnx\"\n", + " FACE_SWAPPER = insightface.model_zoo.get_model(model_path, providers=['CUDAExecutionProvider'])\n", + " return FACE_SWAPPER" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "id": "EnNJl5kDOjrL" + }, + "outputs": [], + "source": [ + "def swap_face(source_face: Face, target_face: Face, temp_frame: Frame) -> Frame:\n", + " return get_face_swapper().get(temp_frame, target_face, source_face, paste_back=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "id": "y0VduuqyaA8R" + }, + "outputs": [], + "source": [ + "def get_face_analyser() -> Any:\n", + " #FACE_ANALYSER.prepare(ctx_id=0, det_size=(640, 640))\n", + " return FACE_ANALYSER" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "id": "ZTHjQ0p2aI9c" + }, + "outputs": [], + "source": [ + "def get_one_face(frame: Frame) -> Any:\n", + " face = get_face_analyser().get(frame)\n", + " try:\n", + " return min(face, key=lambda x: x.bbox[0])\n", + " except ValueError:\n", + " return None\n" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "id": "ygR97Za2aTVO" + }, + "outputs": [], + "source": [ + "def get_many_faces(frame: Frame) -> Any:\n", + " try:\n", + " return get_face_analyser().get(frame)\n", + " except IndexError:\n", + " return None" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "id": "05SG37Fbwip1" + }, + "outputs": [], + "source": [ + "def process_frame(source_face: Face, temp_frame: Frame,manyface: bool) -> Frame:\n", + "\n", + " if manyface:\n", + " many_faces = get_many_faces(temp_frame)\n", + " if many_faces:\n", + " for target_face in many_faces:\n", + " temp_frame = swap_face(source_face, target_face, temp_frame)\n", + " else:\n", + " target_face = get_one_face(temp_frame)\n", + " if target_face:\n", + " temp_frame = swap_face(source_face, target_face, temp_frame)\n", + " return temp_frame" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "id": "znqUf8Gnq4dJ" + }, + "outputs": [], + "source": [ + "# Input and output ports for communication\n", + "local_in_source = 5555\n", + "local_in_temp = 5556\n", + "local_out_frame = 5557" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FehUyVNvGGgZ" + }, + "source": [ + "### [Optional] For Live Streaming\n", + "For Live Streaming from webcam run this cell but for just image swap run the next cell. \n", + "\n", + "Note: Don't run both cells at same time. You should either run this or the one below\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "VYQRVPsdxizn", + "outputId": "def84ce6-46cf-401f-8e60-f1f0216b2975" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PULL one socket bound to tcp://127.0.0.1:5555\n", + "OutputStream ffmpeg bound to tcp://127.0.0.1:5557?listen\n", + "InputStream ffmpeg bound from tcp://127.0.0.1:5556?listen\n" + ] + } + ], + "source": [ + "import zmq\n", + "import threading\n", + "import cv2\n", + "import numpy as np\n", + "import msgpack\n", + "import queue\n", + "import time\n", + "import zlib\n", + "from tqdm import tqdm\n", + "import subprocess\n", + "from collections import deque\n", + "#import matplotlib.pyplot as plt\n", + "\n", + "def create_demo_image():\n", + " # Create a demo image (e.g., a solid color or pattern)\n", + " demo_image = np.zeros((540, 960, 3), dtype=np.uint8) # Black image\n", + " cv2.putText(demo_image, 'Demo Image', (50, 240), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)\n", + " return demo_image\n", + "\n", + "def pull_socket(local_in_port):\n", + " context = zmq.Context()\n", + " socket = context.socket(zmq.REP)\n", + " socket.setsockopt(zmq.RCVHWM, 100000)\n", + " socket.setsockopt(zmq.LINGER, 0)\n", + " address = f\"tcp://127.0.0.1:{local_in_port}\"\n", + " socket.bind(address) # Binding to a different local port\n", + " print(f\"PULL one socket bound to {address}\")\n", + " return socket\n", + "\n", + "\n", + "# Compress image\n", + "def compress_image(image, quality=95):\n", + " # Set the JPEG quality parameter\n", + " encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), quality]\n", + " # Encode the image as a JPEG\n", + " result, encimg = cv2.imencode('.jpg', image, encode_param)\n", + "\n", + " if not result:\n", + " raise Exception(\"Image encoding failed\")\n", + "\n", + " # Decode the encoded image back to an image format\n", + " decimg = cv2.imdecode(encimg, 1)\n", + " return decimg\n", + "# Decompress image\n", + "def decompress_image(encimg):\n", + " image = cv2.imdecode(np.frombuffer(encimg, np.uint8), cv2.IMREAD_COLOR)\n", + " return image\n", + "\n", + "# Global variables\n", + "frames_array=deque(maxlen=2000)\n", + "source_frame = None\n", + "is_manyFace = None\n", + "frameSize = '960x540'\n", + "fps = None\n", + "\n", + "#functions\n", + "def pull_worker(pull_socket):\n", + " global source_frame,is_manyFace,frameSize,fps\n", + " while True:\n", + " try:\n", + " # Receive the JSON with total chunks\n", + " meta_data_json = pull_socket.recv_json()\n", + " #print(meta_data_json)\n", + " total_chunk = meta_data_json['total_chunk']\n", + " # Send acknowledgment for metadata\n", + " pull_socket.send_string(\"ACK\")\n", + " # Receive the array bytes\n", + " source_array_bytes =b''\n", + " for i in range(total_chunk):\n", + " chunk = pull_socket.recv()\n", + " source_array_bytes += chunk\n", + " pull_socket.send_string(f\"ACK {i + 1}/{total_chunk}\")\n", + "\n", + "\n", + " end_message = pull_socket.recv()\n", + " if end_message == b\"END\":\n", + " pull_socket.send_string(\"Final ACK\")\n", + "\n", + " # Deserialize the bytes back to an ndarray\n", + " source_array = np.frombuffer(source_array_bytes, dtype=np.dtype(meta_data_json['dtype_source'])).reshape(meta_data_json['shape_source'])\n", + "\n", + " #plt.imshow(source_array[:, :, ::-1])\n", + " #plt.show()\n", + " #frame_queue.append([\"source\", source_array])\n", + " source_frame = source_array\n", + " is_manyFace = meta_data_json['manyface']\n", + " frameSize = meta_data_json['size']\n", + " fps = meta_data_json[\"fps\"]\n", + "\n", + "\n", + " #process_queue.put((\"source\", source_array))\n", + " break\n", + " except zmq.Again:\n", + " # Sleep briefly to avoid busy-waiting\n", + " time.sleep(0.01)\n", + " except Exception as e:\n", + " print(f\"Error: {e}\")\n", + "def pull_worker_two(local_in_temp):\n", + " ffmpeg_receive_command = [\n", + " 'ffmpeg',\n", + " '-i',f'tcp://127.0.0.1:{local_in_temp}?listen',\n", + " '-f','rawvideo',\n", + " '-pix_fmt','bgr24',\n", + " '-s','960x540',\n", + " 'pipe:1'\n", + " ]\n", + " ffmpeg_receive_process = subprocess.Popen((ffmpeg_receive_command), stdout=subprocess.PIPE)\n", + " timefame =1/25\n", + " print(f\"InputStream ffmpeg bound from tcp://127.0.0.1:{local_in_temp}?listen\")\n", + " global source_frame,is_manyFace\n", + " while True:\n", + " try:\n", + "\n", + " # Receive the JSON with total chunks\n", + " # Read decoded frame from FFmpeg\n", + " raw_frame = ffmpeg_receive_process.stdout.read(960 * 540 * 3)\n", + " if not raw_frame:\n", + " break\n", + " framex = np.frombuffer(raw_frame, dtype=np.uint8).reshape((540, 960, 3))\n", + " #print(framex)\n", + " source_array = source_frame\n", + " is_many_face = is_manyFace\n", + "\n", + " if source_array is not None:\n", + " processed_array = process_frame(get_one_face(source_array),framex,is_many_face)\n", + " #plt.imshow(processed_array[:, :, ::-1])\n", + " #plt.show()\n", + " frames_array.append(processed_array)\n", + "\n", + " time.sleep(timefame)\n", + "\n", + "\n", + " except zmq.Again:\n", + " # Sleep briefly to avoid busy-waiting\n", + " time.sleep(0.01)\n", + " except Exception as e:\n", + " print(f\"Error: {e}\")\n", + "def push_worker(local_out_frame):\n", + " source_array = None\n", + " temp_array = None\n", + " is_many_face = None\n", + " global source_frame,is_manyFace,frameSize,fps\n", + " print(f\"OutputStream ffmpeg bound to tcp://127.0.0.1:{local_out_frame}?listen\")\n", + " ffmpeg_encode_command = [\n", + " 'ffmpeg',\n", + " '-f', 'rawvideo',\n", + " '-pix_fmt', 'bgr24',\n", + " '-s', '960x540',\n", + " '-r', '5',\n", + " '-i', 'pipe:',\n", + " #'-vf', 'fps=5',\n", + " #'-c:v', 'libx264',\n", + " #'-probesize', '32',\n", + " #'-analyzeduration', '0',\n", + " '-f', 'mpegts',\n", + " f'tcp://127.0.0.1:{local_out_frame}?listen'\n", + " ]\n", + " timefame =1/5\n", + "\n", + " ffmpeg_encode_process = subprocess.Popen((ffmpeg_encode_command), stdin=subprocess.PIPE)\n", + "\n", + "\n", + " demo_image = create_demo_image()\n", + " frames_to_skip = 200 # Number of frames to skip to reduce delay\n", + " frame_count =0\n", + " wait_frame = 100\n", + " try:\n", + " while True:\n", + "\n", + " # Get the processed array and metadata from the queue\n", + " if len(frames_array)>wait_frame:\n", + "\n", + " while len(frames_array)>7:#frame_count0:\n", + " #print(frames_array)\n", + " item = frames_array.popleft()#process_queue.get()\n", + " if item[0]==\"source\":\n", + " source_frm = item[1]\n", + " is_manyface = item[2]\n", + " if item[0]==\"temp\":\n", + " temp_frm = item[1]\n", + " print(\"Recieved\")\n", + "\n", + " if temp_frm is not None and source_frm is not None:\n", + "\n", + " processed_frm =process_frame(get_one_face(source_frm),temp_frm,is_manyface)\n", + "\n", + " face_bytes = processed_frm.tobytes()\n", + " chunk_size = 1024*200\n", + " total_chunk = len(face_bytes) // chunk_size + 1\n", + " metadata ={\n", + "\n", + " 'dtype_source':str(processed_frm.dtype),\n", + " 'shape_source':processed_frm.shape,\n", + " 'size':'640x480',\n", + " 'fps':'60'\n", + " #'shape_temp':temp_frame.shape\n", + " }\n", + " new_metadata = {'total_chunk': total_chunk}\n", + " metadata.update(new_metadata)\n", + " # Send metadata first\n", + " push_socket.send_json(metadata)\n", + " # Wait for acknowledgment for metadata\n", + " ack = push_socket.recv_string()\n", + " with tqdm(total=total_chunk, desc=\"Sending chunks\", unit=\"chunk\") as pbar:\n", + " for i in range(total_chunk):\n", + " chunk = face_bytes[i * chunk_size:(i + 1) * chunk_size]\n", + " # Send the chunk\n", + " push_socket.send(chunk)\n", + " # Wait for acknowledgment after sending each chunk\n", + " ack = push_socket.recv_string()\n", + " pbar.set_postfix_str(f'Chunk {i + 1}/{total_chunk} ack: {ack}')\n", + " pbar.update(1)\n", + "\n", + " # Send a final message to indicate all chunks are sent\n", + " push_socket.send(b\"END\")\n", + " # Wait for the final reply\n", + " final_reply_message = push_socket.recv_string()\n", + " print(f\"Received final reply: {final_reply_message}\")\n", + "\n", + " temp_frm = None\n", + " source_frm = None\n", + " is_manyface = None\n", + "\n", + "\n", + " except Exception as e:\n", + " print (f\"Error in Push Wokr {e}\")\n", + "\n", + "local_in_source = 5555\n", + "local_in_temp = 5556\n", + "local_out_frame = 5557\n", + "\n", + "# Create sockets\n", + "pull_socket_ = pull_socket(local_in_source)\n", + "pull_socket_two = pull_socket(local_in_temp)\n", + "push_socket_ = push_socket(local_out_frame)\n", + "# Run both workers in separate threads\n", + "# Start the pull worker thread\n", + "pull_thread = threading.Thread(target=pull_worker, args=(pull_socket_,))\n", + "pull_thread.start()\n", + "# Start the push worker thread\n", + "pull_thread_two = threading.Thread(target=pull_worker_two, args=(pull_socket_two,))\n", + "pull_thread_two.start()\n", + "# Start the push worker thread\n", + "push_thread = threading.Thread(target=push_worker,args=(push_socket_,))\n", + "push_thread.start()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "a9JW1rhH3H-T" + }, + "source": [ + "### [Optional] Open FRP tunnel" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "VNDmeHcx3Zrx" + }, + "outputs": [], + "source": [ + "\n", + "frpc_config = f\"\"\"serverAddr = \"194.113.64.71\"\n", + "serverPort = 7000\n", + "\n", + "[[proxies]]\n", + "name = \"Pull-tcp\"\n", + "type = \"tcp\"\n", + "localIP = \"127.0.0.1\"\n", + "localPort = {local_in_source}\n", + "remotePort = 6000\n", + "\n", + "[[proxies]]\n", + "name = \"Pull-tcp_two\"\n", + "type = \"tcp\"\n", + "localIP = \"127.0.0.1\"\n", + "localPort = {local_in_temp}\n", + "remotePort = 6001\n", + "\n", + "[[proxies]]\n", + "name = \"Push-tcp\"\n", + "type = \"tcp\"\n", + "localIP = \"127.0.0.1\"\n", + "localPort = {local_out_frame}\n", + "remotePort = 6002\"\"\"\n", + "with open('frp_0.59.0_linux_amd64/frpc.toml', 'w') as f:\n", + " f.write(frpc_config)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Remote tcp\n", + "print(f'tcp://194.113.64.71:6000 ---> {local_in_source}')\n", + "print(f'tcp://194.113.64.71:6001 ---> {local_in_temp}')\n", + "print(f'tcp://194.113.64.71:6002 ---> {local_out_frame}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Ucjrp4fM3mQ4" + }, + "outputs": [], + "source": [ + "from subprocess import Popen, PIPE\n", + "import time\n", + "ps = Popen('/content/deepfakecollab/Scripts/open_tunnel_frs.sh', stdout=PIPE, stderr=PIPE)\n", + "time.sleep(3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "SDFT4XXJkVxT", + "outputId": "1738d849-a3e1-403d-95d0-d3a4e4f6cd44" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Opening tunnel\n", + "\u001b[1;34m2024-08-03 20:10:33.967 [I] [sub/root.go:142] start frpc service for config file [frp_0.59.0_linux_amd64/frpc.toml]\n", + "\u001b[0m\u001b[1;34m2024-08-03 20:10:33.967 [I] [client/service.go:294] try to connect to server...\n", + "\u001b[0m\u001b[1;34m2024-08-03 20:10:34.377 [I] [client/service.go:286] [b793872517fab479] login to server success, get run id [b793872517fab479]\n", + "\u001b[0m\u001b[1;34m2024-08-03 20:10:34.377 [I] [proxy/proxy_manager.go:173] [b793872517fab479] proxy added: [Pull-tcp Pull-tcp_two Push-tcp]\n", + "\u001b[0m\u001b[1;34m2024-08-03 20:10:34.514 [I] [client/control.go:168] [b793872517fab479] [Pull-tcp] start proxy success\n", + "\u001b[0m\u001b[1;34m2024-08-03 20:10:34.514 [I] [client/control.go:168] [b793872517fab479] [Pull-tcp_two] start proxy success\n", + "\u001b[0m\u001b[1;34m2024-08-03 20:10:34.514 [I] [client/control.go:168] [b793872517fab479] [Push-tcp] start proxy success\n", + "\u001b[0m^C\n" + ] + } + ], + "source": [ + "!/content/deepfakecollab/Scripts/open_tunnel_frs.sh" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "buCnH_YpxKWa" + }, + "source": [ + "### [Optional] Open Ngrok tunnel" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "id": "HyY2B5bmxlui" + }, + "outputs": [], + "source": [ + "from subprocess import Popen, PIPE\n", + "import shlex\n", + "import json\n", + "import time\n", + "\n", + "\n", + "def run_with_pipe(command):\n", + " commands = list(map(shlex.split,command.split(\"|\")))\n", + " ps = Popen(commands[0], stdout=PIPE, stderr=PIPE)\n", + " for command in commands[1:]:\n", + " ps = Popen(command, stdin=ps.stdout, stdout=PIPE, stderr=PIPE)\n", + " return ps.stdout.readlines()\n", + "\n", + "\n", + "def get_tunnel_adresses():\n", + " info = run_with_pipe(\"curl http://localhost:4040/api/tunnels\")\n", + " assert info\n", + "\n", + " info = json.loads(info[0])\n", + " for tunnel in info['tunnels']:\n", + " url = tunnel['public_url']\n", + " port = url.split(':')[-1]\n", + " local_port = tunnel['config']['addr'].split(':')[-1]\n", + " print(f'{url} -> {local_port} [{tunnel[\"name\"]}]')\n", + " if tunnel['name'] == 'input':\n", + " in_addr = url\n", + " elif tunnel['name'] == 'inputtwo':\n", + " in_addrtwo = url\n", + " elif tunnel['name'] == 'output':\n", + " out_addr = url\n", + " else:\n", + " print(f'unknown tunnel: {tunnel[\"name\"]}')\n", + "\n", + " return in_addr,in_addrtwo, out_addr" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "id": "Lnqdi11sxGGR" + }, + "outputs": [], + "source": [ + "config =\\\n", + "f\"\"\"\n", + "version: 2\n", + "authtoken: {authtoken}\n", + "region: {region}\n", + "console_ui: False\n", + "tunnels:\n", + " input:\n", + " addr: {local_in_source}\n", + " proto: tcp\n", + " inputtwo:\n", + " addr: {local_in_temp}\n", + " proto: tcp\n", + " output:\n", + " addr: {local_out_frame}\n", + " proto: tcp\n", + "\"\"\"\n", + "\n", + "with open('ngrok.conf', 'w') as f:\n", + " f.write(config)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "id": "bQiD7DmexYvy" + }, + "outputs": [], + "source": [ + "# (Re)Open tunnel\n", + "ps = Popen('/content/deepfakecollab/Scripts/open_tunnel_ngrok.sh', stdout=PIPE, stderr=PIPE)\n", + "time.sleep(3)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "4kC3axJEx5e8", + "outputId": "21867038-e5c4-4038-9876-c41a569b09e7" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "tcp://0.tcp.eu.ngrok.io:11640 -> 5556 [inputtwo]\n", + "tcp://4.tcp.eu.ngrok.io:16503 -> 5555 [input]\n", + "tcp://0.tcp.eu.ngrok.io:13155 -> 5557 [output]\n", + "Tunnel opened\n" + ] + } + ], + "source": [ + "# Get tunnel addresses\n", + "try:\n", + " in_addr,in_addr_two, out_addr = get_tunnel_adresses()\n", + " print(\"Tunnel opened\")\n", + "except Exception as e:\n", + " [print(l.decode(), end='') for l in ps.stdout.readlines()]\n", + " print(\"Something went wrong, reopen the tunnel\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "F9Qs1U8gu-l3" + }, + "source": [ + "### **DeBugging (Ignore)**" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "d_gTh_hfbstK" + }, + "source": [ + "### View Source Image" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "otrLL3NnHCt3", + "outputId": "60a822ea-b55c-4f31-cf19-ddc4390de81f" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "False\n" + ] + } + ], + "source": [ + "print(is_manyFace)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "9Kqc-HRYwa08", + "outputId": "0ff9e514-b029-4df2-c52f-230ddd9dd2a6" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\n" + ] + } + ], + "source": [ + "print(len(frames_array))" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [ + "GYFbuuCdvTaO", + "0rTCd2R1uw7A", + "9n-qESinvM22", + "JmNGrasvwnVX", + "sNHGT2FjjZiK", + "DTbmyO-CK-8b", + "FehUyVNvGGgZ", + "Eb_cbyoaHKrT", + "a9JW1rhH3H-T", + "buCnH_YpxKWa", + "d_gTh_hfbstK" + ], + "gpuType": "L4", + "machine_shape": "hm", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/modules/core.py b/modules/core.py index db64f37..969623e 100644 --- a/modules/core.py +++ b/modules/core.py @@ -34,7 +34,7 @@ def parse_args() -> None: program.add_argument('-s', '--source', help='select an source image', dest='source_path') program.add_argument('-t', '--target', help='select an target image or video', dest='target_path') program.add_argument('-o', '--output', help='select output file or directory', dest='output_path') - program.add_argument('--frame-processor', help='pipeline of frame processors', dest='frame_processor', default=['face_swapper'], choices=['face_swapper', 'face_enhancer'], nargs='+') + program.add_argument('--frame-processor', help='pipeline of frame processors', dest='frame_processor', default=['face_swapper'], choices=['face_swapper', 'face_enhancer','remote_processor'], nargs='+') program.add_argument('--keep-fps', help='keep original fps', dest='keep_fps', action='store_true', default=False) program.add_argument('--keep-audio', help='keep original audio', dest='keep_audio', action='store_true', default=True) program.add_argument('--keep-frames', help='keep temporary frames', dest='keep_frames', action='store_true', default=False) @@ -74,7 +74,11 @@ def parse_args() -> None: modules.globals.fp_ui['face_enhancer'] = True else: modules.globals.fp_ui['face_enhancer'] = False - + #for Remote tumbler: + if 'remote_processor' in args.frame_processor: + modules.globals.fp_ui['remote_processor'] = True + else: + modules.globals.fp_ui['remote_processor'] = False modules.globals.nsfw = False # translate deprecated args diff --git a/modules/globals.py b/modules/globals.py index c392a80..86c14d4 100644 --- a/modules/globals.py +++ b/modules/globals.py @@ -27,4 +27,7 @@ log_level = 'error' fp_ui: Dict[str, bool] = {} nsfw = None camera_input_combobox = None -webcam_preview_running = False \ No newline at end of file +webcam_preview_running = False +push_addr = None +pull_addr = None +push_addr_two = None \ No newline at end of file diff --git a/modules/metadata.py b/modules/metadata.py index 9309bea..c8d7738 100644 --- a/modules/metadata.py +++ b/modules/metadata.py @@ -1,3 +1,3 @@ name = 'Deep Live Cam' -version = '1.3.0' -edition = 'Portable' +version = '1.3.1' +edition = 'Portable-Colab' diff --git a/modules/processors/frame/remote_processor.py b/modules/processors/frame/remote_processor.py new file mode 100644 index 0000000..1e42567 --- /dev/null +++ b/modules/processors/frame/remote_processor.py @@ -0,0 +1,243 @@ +import zmq +import cv2 +import modules.globals +import numpy as np +import threading +import time +import io +from tqdm import tqdm +from modules.typing import Face, Frame +from typing import Any,List +from modules.core import update_status +from modules.utilities import conditional_download, resolve_relative_path, is_image, is_video +import zlib +import subprocess +from cv2 import VideoCapture +import queue +NAME = 'DLC.REMOTE-PROCESSOR' + +context = zmq.Context() + +# Socket to send messages on +def push_socket(address) -> zmq.Socket: + sender_sock = context.socket(zmq.REQ) + sender_sock.connect(address) + return sender_sock +def pull_socket(address) -> zmq.Socket: + sender_sock = context.socket(zmq.REP) + sender_sock.connect(address) + return sender_sock + +def pre_check() -> bool: + if not modules.globals.push_addr and not modules.globals.pull_addr: + return False + return True + + +def pre_start() -> bool: + if not is_image(modules.globals.target_path) and not is_video(modules.globals.target_path): + update_status('Select an image or video for target path.', NAME) + return False + return True + +def stream_frame(temp_frame: Frame,stream_out: subprocess.Popen[bytes],stream_in: subprocess.Popen[bytes]) -> Frame: + temp_framex = swap_face_remote(temp_frame,stream_out,stream_in) + + return temp_framex + +def process_frame(source_frame: Frame, temp_frame: Frame)-> Frame: + temp_framex = swap_frame_face_remote(source_frame,temp_frame) + + return temp_framex +def send_data(sender: zmq.Socket, face_bytes: bytes, metadata: dict, address: str) -> None: + chunk_size = 1024*100 + total_chunk = len(face_bytes) // chunk_size + 1 + new_metadata = {'total_chunk': total_chunk} + metadata.update(new_metadata) + # Send metadata first + sender.send_json(metadata) + # Wait for acknowledgment for metadata + ack = sender.recv_string() + with tqdm(total=total_chunk, desc="Sending chunks", unit="chunk") as pbar: + for i in range(total_chunk): + chunk = face_bytes[i * chunk_size:(i + 1) * chunk_size] + # Send the chunk + sender.send(chunk) + # Wait for acknowledgment after sending each chunk + ack = sender.recv_string() + pbar.set_postfix_str(f'Chunk {i + 1}/{total_chunk} ack: {ack}') + pbar.update(1) + + # Send a final message to indicate all chunks are sent + sender.send(b"END") + # Wait for the final reply + final_reply_message = sender.recv_string() + print(f"Received final reply: {final_reply_message}") + +def send_source_frame(source_face: Frame)-> None: + sender = push_socket(modules.globals.push_addr) + source_face_bytes = source_face.tobytes() + metadata = { + 'manyface':(modules.globals.many_faces), + 'dtype_source':str(source_face.dtype), + 'shape_source':source_face.shape, + 'size':'640x480', + 'fps':'60' + #'shape_temp':temp_frame.shape + } + send_data(sender, source_face_bytes, metadata,modules.globals.push_addr) + +def send_temp_frame(temp_face: Frame)-> None: + sender = push_socket(modules.globals.push_addr_two) + source_face_bytes = temp_face.tobytes() + metadata = { + 'manyface':(modules.globals.many_faces), + 'dtype_temp':str(temp_face.dtype), + 'shape_temp':temp_face.shape, + + #'shape_temp':temp_frame.shape + } + send_data(sender, source_face_bytes, metadata,modules.globals.push_addr) + +def receive_processed_frame(output_queue: queue.Queue)-> None: + while True: + pull_socket_ = pull_socket(modules.globals.pull_addr) + meta_data_json = pull_socket_.recv_json() + print(meta_data_json) + total_chunk = meta_data_json['total_chunk'] + # Send acknowledgment for metadata + pull_socket_.send_string("ACK") + # Receive the array bytes + source_array_bytes =b'' + with tqdm(total=total_chunk, desc="Receiving chunks", unit="chunk") as pbar: + for i in range(total_chunk): + chunk = pull_socket_.recv() + source_array_bytes += chunk + pull_socket_.send_string(f"ACK {i + 1}/{total_chunk}") + pbar.set_postfix_str(f'Chunk {i + 1}/{total_chunk}') + pbar.update(1) + + + end_message = pull_socket_.recv() + if end_message == b"END": + pull_socket_.send_string("Final ACK") + + # Deserialize the bytes back to an ndarray + source_array = np.frombuffer(source_array_bytes, dtype=np.dtype(meta_data_json['dtype_source'])).reshape(meta_data_json['shape_source']) + + output_queue.put(source_array) + break +def send_streams(cap: VideoCapture) -> subprocess.Popen[bytes]: + + ffmpeg_command = [ + 'ffmpeg', + '-f', 'rawvideo', + '-pix_fmt', 'bgr24', + '-s', f"{int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))}x{int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))}", + '-r', str(int(cap.get(cv2.CAP_PROP_FPS))), + '-i', '-', + '-c:v', 'libx264', + '-preset', 'ultrafast', + '-tune', 'zerolatency', + '-fflags', 'nobuffer', + '-flags', 'low_delay', + '-rtbufsize', '100M', + '-f', 'mpegts', modules.globals.push_addr_two #'tcp://127.0.0.1:5552' + ] + + + ffmpeg_process = subprocess.Popen(ffmpeg_command, stdin=subprocess.PIPE) + return ffmpeg_process +def recieve_streams(cap: VideoCapture)->subprocess.Popen[bytes]: + ffmpeg_command_recie = [ + 'ffmpeg', + '-i',modules.globals.pull_addr, #'tcp://127.0.0.1:5553', + '-f','rawvideo', + '-pix_fmt','bgr24', + '-s','960x540',#'640x480', + 'pipe:1' + ] + + ffmpeg_process_com = subprocess.Popen(ffmpeg_command_recie, stdout=subprocess.PIPE) + return ffmpeg_process_com + +def write_to_stdin(queue: queue.Queue, stream_out: subprocess.Popen): + + temp_frame = queue.get() + temp_frame_bytes = temp_frame.tobytes() + stream_out.stdin.write(temp_frame_bytes) +def read_from_stdout(queue: queue.Queue, stream_in: subprocess.Popen, output_queue: queue.Queue): + + raw_frame = stream_in.stdout.read(960 * 540 * 3) + + + frame = np.frombuffer(raw_frame, dtype=np.uint8).reshape((540, 960, 3)) + output_queue.put(frame) +def swap_face_remote(temp_frame: Frame,stream_out:subprocess.Popen[bytes],stream_in: subprocess.Popen[bytes]) -> Frame: + input_queue = queue.Queue() + output_queue = queue.Queue() + + # Start threads for stdin and stdout + write_thread = threading.Thread(target=write_to_stdin, args=(input_queue, stream_out)) + read_thread = threading.Thread(target=read_from_stdout, args=(input_queue, stream_in, output_queue)) + + write_thread.start() + read_thread.start() + + # Send the frame to the stdin thread + input_queue.put(temp_frame) + + # Wait for the processed frame from the stdout thread + processed_frame = output_queue.get() + + # Stop the threads + input_queue.put(None) + write_thread.join() + read_thread.join() + + return processed_frame + + +def swap_frame_face_remote(source_frame: Frame,temp_frame: Frame) -> Frame: + #input_queue = queue.Queue() + output_queue = queue.Queue() + + # Start threads for stdin and stdout + write_thread = threading.Thread(target=send_source_frame, args=(source_frame,)) + write_thread_tw = threading.Thread(target=send_temp_frame, args=(temp_frame,)) + read_thread_ = threading.Thread(target=receive_processed_frame, args=(output_queue,)) + + write_thread.start() + write_thread_tw.start() + read_thread_.start() + + # Send the frame to the stdin thread + + # Wait for the processed frame from the stdout thread + processed_frame = output_queue.get() + + # Stop the threads + write_thread.join() + write_thread_tw.join() + read_thread_.join() + + return processed_frame + + +def process_frames(source_path: str, temp_frame_paths: List[str], progress: Any = None) -> None: + for temp_frame_path in temp_frame_paths: + temp_frame = cv2.imread(temp_frame_path) + result = process_frame(None, temp_frame) + cv2.imwrite(temp_frame_path, result) + if progress: + progress.update(1) + + +def process_image(source_path: str, target_path: str, output_path: str) -> None: + target_frame = cv2.imread(target_path) + result = process_frame(None, target_frame) + cv2.imwrite(output_path, result) + + +def process_video(source_path: str, temp_frame_paths: List[str]) -> None: + modules.processors.frame.core.process_video(None, temp_frame_paths, process_frames) diff --git a/modules/ui.py b/modules/ui.py index 2759a9e..d6d3363 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -1,6 +1,8 @@ +import sys import os import webbrowser import customtkinter as ctk +import tkinter as tk from typing import Callable, Tuple import cv2 from PIL import Image, ImageOps @@ -14,7 +16,7 @@ from modules.utilities import is_image, is_video, resolve_relative_path ROOT = None ROOT_HEIGHT = 700 -ROOT_WIDTH = 600 +ROOT_WIDTH = 800 PREVIEW = None PREVIEW_MAX_HEIGHT = 700 @@ -62,47 +64,80 @@ def create_root(start: Callable[[], None], destroy: Callable[[], None]) -> ctk.C target_label.place(relx=0.6, rely=0.1, relwidth=0.3, relheight=0.25) select_face_button = ctk.CTkButton(root, text='Select a face', cursor='hand2', command=lambda: select_source_path()) - select_face_button.place(relx=0.1, rely=0.4, relwidth=0.3, relheight=0.1) + select_face_button.place(relx=0.1, rely=0.3, relwidth=0.3, relheight=0.1) select_target_button = ctk.CTkButton(root, text='Select a target', cursor='hand2', command=lambda: select_target_path()) - select_target_button.place(relx=0.6, rely=0.4, relwidth=0.3, relheight=0.1) + select_target_button.place(relx=0.6, rely=0.3, relwidth=0.3, relheight=0.1) keep_fps_value = ctk.BooleanVar(value=modules.globals.keep_fps) keep_fps_checkbox = ctk.CTkSwitch(root, text='Keep fps', variable=keep_fps_value, cursor='hand2', command=lambda: setattr(modules.globals, 'keep_fps', not modules.globals.keep_fps)) - keep_fps_checkbox.place(relx=0.1, rely=0.6) + keep_fps_checkbox.place(relx=0.1, rely=0.5) keep_frames_value = ctk.BooleanVar(value=modules.globals.keep_frames) keep_frames_switch = ctk.CTkSwitch(root, text='Keep frames', variable=keep_frames_value, cursor='hand2', command=lambda: setattr(modules.globals, 'keep_frames', keep_frames_value.get())) - keep_frames_switch.place(relx=0.1, rely=0.65) + keep_frames_switch.place(relx=0.1, rely=0.55) # for FRAME PROCESSOR ENHANCER tumbler: enhancer_value = ctk.BooleanVar(value=modules.globals.fp_ui['face_enhancer']) enhancer_switch = ctk.CTkSwitch(root, text='Face Enhancer', variable=enhancer_value, cursor='hand2', command=lambda: update_tumbler('face_enhancer',enhancer_value.get())) - enhancer_switch.place(relx=0.1, rely=0.7) + enhancer_switch.place(relx=0.1, rely=0.6) + + remote_process_value = ctk.BooleanVar(value=modules.globals.fp_ui['remote_processor']) + remote_process_switch = ctk.CTkSwitch(root, text='Remote Processor', variable=remote_process_value, cursor='hand2', command=lambda: update_tumbler('remote_processor',remote_process_value.get())) + remote_process_switch.place(relx=0.1, rely=0.65) + + def on_text_change(event=None): + setattr(modules.globals, 'pull_addr', text_box_addr_in.get("1.0", tk.END).strip()) + + def on_text_change_out(event=None): + setattr(modules.globals, 'push_addr', text_box_addr_out.get("1.0", tk.END).strip()) + def on_text_change_out_two(event=None): + setattr(modules.globals, 'push_addr_two', text_box_addr_out_t.get("1.0", tk.END).strip()) keep_audio_value = ctk.BooleanVar(value=modules.globals.keep_audio) keep_audio_switch = ctk.CTkSwitch(root, text='Keep audio', variable=keep_audio_value, cursor='hand2', command=lambda: setattr(modules.globals, 'keep_audio', keep_audio_value.get())) - keep_audio_switch.place(relx=0.6, rely=0.6) + keep_audio_switch.place(relx=0.6, rely=0.5) many_faces_value = ctk.BooleanVar(value=modules.globals.many_faces) many_faces_switch = ctk.CTkSwitch(root, text='Many faces', variable=many_faces_value, cursor='hand2', command=lambda: setattr(modules.globals, 'many_faces', many_faces_value.get())) - many_faces_switch.place(relx=0.6, rely=0.65) + many_faces_switch.place(relx=0.6, rely=0.55) -# nsfw_value = ctk.BooleanVar(value=modules.globals.nsfw) -# nsfw_switch = ctk.CTkSwitch(root, text='NSFW', variable=nsfw_value, cursor='hand2', command=lambda: setattr(modules.globals, 'nsfw', nsfw_value.get())) -# nsfw_switch.place(relx=0.6, rely=0.7) + nsfw_value = ctk.BooleanVar(value=modules.globals.nsfw) + nsfw_switch = ctk.CTkSwitch(root, text='NSFW', variable=nsfw_value, cursor='hand2', command=lambda: setattr(modules.globals, 'nsfw', nsfw_value.get())) + nsfw_switch.place(relx=0.6, rely=0.6) + + #Label a text box + label = ctk.CTkLabel(root, text="In:") + label.place(relx=0.1, rely=0.72, anchor=tk.E) + #Label a text box + label = ctk.CTkLabel(root, text="OutS:") + label.place(relx=0.6, rely=0.72, anchor=tk.E) + # Create a text box + text_box_addr_in = ctk.CTkTextbox(root, width=200, height=10) + text_box_addr_in.place(relx=0.1, rely=0.7) + text_box_addr_in.bind("", on_text_change) + + # Create a text box + text_box_addr_out = ctk.CTkTextbox(root, width=100, height=10) + text_box_addr_out.place(relx=0.6, rely=0.7) + text_box_addr_out.bind("", on_text_change_out) + + # Create a text box + text_box_addr_out_t = ctk.CTkTextbox(root, width=100, height=10) + text_box_addr_out_t.place(relx=0.8, rely=0.7) + text_box_addr_out_t.bind("", on_text_change_out_two) start_button = ctk.CTkButton(root, text='Start', cursor='hand2', command=lambda: select_output_path(start)) - start_button.place(relx=0.15, rely=0.80, relwidth=0.2, relheight=0.05) + start_button.place(relx=0.15, rely=0.75, relwidth=0.2, relheight=0.05) stop_button = ctk.CTkButton(root, text='Destroy', cursor='hand2', command=lambda: destroy()) - stop_button.place(relx=0.4, rely=0.80, relwidth=0.2, relheight=0.05) + stop_button.place(relx=0.4, rely=0.75, relwidth=0.2, relheight=0.05) preview_button = ctk.CTkButton(root, text='Preview', cursor='hand2', command=lambda: toggle_preview()) - preview_button.place(relx=0.65, rely=0.80, relwidth=0.2, relheight=0.05) + preview_button.place(relx=0.65, rely=0.75, relwidth=0.2, relheight=0.05) live_button = ctk.CTkButton(root, text='Live', cursor='hand2', command=lambda: webcam_preview()) - live_button.place(relx=0.40, rely=0.86, relwidth=0.2, relheight=0.05) + live_button.place(relx=0.40, rely=0.85, relwidth=0.2, relheight=0.05) status_label = ctk.CTkLabel(root, text=None, justify='center') status_label.place(relx=0.1, rely=0.9, relwidth=0.8) @@ -235,15 +270,28 @@ def init_preview() -> None: def update_preview(frame_number: int = 0) -> None: if modules.globals.source_path and modules.globals.target_path: temp_frame = get_video_frame(modules.globals.target_path, frame_number) + remote_process=False if modules.globals.nsfw == False: from modules.predicter import predict_frame if predict_frame(temp_frame): quit() for frame_processor in get_frame_processors_modules(modules.globals.frame_processors): - temp_frame = frame_processor.process_frame( - get_one_face(cv2.imread(modules.globals.source_path)), - temp_frame - ) + if 'remote_processor' in modules.globals.frame_processors : + remote_process = True + if frame_processor.__name__ =="modules.processors.frame.remote_processor": + print('------- Remote Process ----------') + source_data = cv2.imread(modules.globals.source_path) + if not frame_processor.pre_check(): + print("No Input and Output Address") + sys.exit() + temp_frame=frame_processor.process_frame(source_data,temp_frame) + + + if not remote_process: + temp_frame = frame_processor.process_frame( + get_one_face(cv2.imread(modules.globals.source_path)), + temp_frame + ) image = Image.fromarray(cv2.cvtColor(temp_frame, cv2.COLOR_BGR2RGB)) image = ImageOps.contain(image, (PREVIEW_MAX_WIDTH, PREVIEW_MAX_HEIGHT), Image.LANCZOS) image = ctk.CTkImage(image, size=image.size) @@ -257,6 +305,7 @@ def webcam_preview(): global preview_label, PREVIEW cap = cv2.VideoCapture(0) # Use index for the webcam (adjust the index accordingly if necessary) + cap.set(cv2.CAP_PROP_BUFFERSIZE,3) cap.set(cv2.CAP_PROP_FRAME_WIDTH, 960) # Set the width of the resolution cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 540) # Set the height of the resolution cap.set(cv2.CAP_PROP_FPS, 60) # Set the frame rate of the webcam @@ -270,30 +319,63 @@ def webcam_preview(): frame_processors = get_frame_processors_modules(modules.globals.frame_processors) source_image = None # Initialize variable for the selected face image + remote_process=False # By default remote process is set to disabled + stream_out = None # Both veriable stores the subprocess runned by ffmpeg + stream_in = None - while True: - ret, frame = cap.read() - if not ret: - break + if 'remote_processor' in modules.globals.frame_processors : + remote_modules = get_frame_processors_modules(['remote_processor']) + source_data = cv2.imread(modules.globals.source_path) + remote_modules[1].send_source_frame(source_data) + #start ffmpeg stream out subprocess + stream_out = remote_modules[1].send_streams(cap) + #start ffmpeg stream In subprocess + stream_in = remote_modules[1].recieve_streams(cap) + + remote_process = True + try: + while True: + ret, frame = cap.read() + if not ret: + break - # Select and save face image only once - if source_image is None and modules.globals.source_path: - source_image = get_one_face(cv2.imread(modules.globals.source_path)) + # Select and save face image only once + if source_image is None and modules.globals.source_path: + source_image = get_one_face(cv2.imread(modules.globals.source_path)) - temp_frame = frame.copy() #Create a copy of the frame + temp_frame = frame.copy() #Create a copy of the frame - for frame_processor in frame_processors: - temp_frame = frame_processor.process_frame(source_image, temp_frame) + for frame_processor in frame_processors: + if remote_process: + if frame_processor.__name__ =="modules.processors.frame.remote_processor": + #print('------- Remote Process ----------') + if not frame_processor.pre_check(): + print("No Input and Output Address") + sys.exit() + _frame = frame_processor.stream_frame(temp_frame,stream_out,stream_in) + if _frame is not None: + temp_frame = _frame + - image = cv2.cvtColor(temp_frame, cv2.COLOR_BGR2RGB) # Convert the image to RGB format to display it with Tkinter - image = Image.fromarray(image) - image = ImageOps.contain(image, (PREVIEW_MAX_WIDTH, PREVIEW_MAX_HEIGHT), Image.LANCZOS) - image = ctk.CTkImage(image, size=image.size) - preview_label.configure(image=image) - ROOT.update() - - if PREVIEW.state() == 'withdrawn': - break - - cap.release() - PREVIEW.withdraw() # Close preview window when loop is finished + if not remote_process: + temp_frame = frame_processor.process_frame(source_image, temp_frame) + if not remote_process: + image = cv2.cvtColor(temp_frame, cv2.COLOR_BGR2RGB) # Convert the image to RGB format to display it with Tkinter + image = Image.fromarray(image) + image = ImageOps.contain(image, (PREVIEW_MAX_WIDTH, PREVIEW_MAX_HEIGHT), Image.LANCZOS) + image = ctk.CTkImage(image, size=image.size) + preview_label.configure(image=image) + ROOT.update() + elif _frame is not None: + image = cv2.cvtColor(temp_frame, cv2.COLOR_BGR2RGB) # Convert the image to RGB format to display it with Tkinter + image = Image.fromarray(image) + image = ImageOps.contain(image, (PREVIEW_MAX_WIDTH, PREVIEW_MAX_HEIGHT), Image.LANCZOS) + image = ctk.CTkImage(image, size=image.size) + preview_label.configure(image=image) + ROOT.update() + + if PREVIEW.state() == 'withdrawn': + break + finally: + cap.release() + PREVIEW.withdraw() # Close preview window when loop is finished diff --git a/requirements.txt b/requirements.txt index f65195e..7842964 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,3 +21,4 @@ opennsfw2==0.10.2 protobuf==4.23.2 tqdm==4.66.4 gfpgan==1.3.8 +zmq==26.1.0 \ No newline at end of file