Compare commits

...

10 Commits

Author SHA1 Message Date
Kenneth Estanislao 229375465d
Update README.md
added some credits
2024-09-11 00:05:06 +08:00
Kenneth Estanislao 49d3f9a3cc recommit webcam option 2024-09-11 00:02:45 +08:00
Kenneth Estanislao 39238ee80f
Merge pull request from pereiraroland26/main
Added support for multiple faces
2024-09-10 23:35:19 +08:00
Roland Pereira d7c6226eb7 updated button widths on popup 2024-09-10 18:53:25 +05:30
Roland Pereira eb140e59c2 commiting gitignore 2024-09-10 16:00:24 +05:30
pereiraroland26 f122006024 updated README.md and created variables for pop dimensions 2024-09-10 14:28:33 +05:30
Roland Pereira 0a144ec57f
Merge branch 'hacksider:main' into main 2024-09-10 13:48:40 +05:30
pereiraroland26 da3498c36f Merge branch 'main' of https://github.com/pereiraroland26/Deep-Live-Cam_v2.0 2024-09-10 05:41:46 +05:30
pereiraroland26@gmail.com 53fc65ca7c Added ability to map faces 2024-09-10 05:40:55 +05:30
james 397c84fa8b Added ability to map faces 2024-09-10 04:37:58 +05:30
9 changed files with 354 additions and 36 deletions

2
.gitignore vendored
View File

@ -22,3 +22,5 @@ models/inswapper_128.onnx
models/GFPGANv1.4.pth
*.onnx
models/DMDNet.pth
faceswap/
.vscode/

View File

@ -146,6 +146,7 @@ options:
--keep-audio keep original audio
--keep-frames keep temporary frames
--many-faces process every face
--map-faces map source target faces
--nsfw-filter filter the NSFW image or video
--video-encoder {libx264,libx265,libvpx-vp9} adjust output video encoder
--video-quality [0-51] adjust output video quality
@ -323,6 +324,7 @@ If you want the latest and greatest build, or want to see some new great feature
- [deepinsight](https://github.com/deepinsight): for their [insightface](https://github.com/deepinsight/insightface) project which provided a well-made library and models. Please be reminded that the [use of the model is for non-commercial research purposes only](https://github.com/deepinsight/insightface?tab=readme-ov-file#license).
- [havok2-htwo](https://github.com/havok2-htwo) : for sharing the code for webcam
- [GosuDRM](https://github.com/GosuDRM/nsfw-roop) : for uncensoring roop
- [pereiraroland26](https://github.com/pereiraroland26) : Multiple faces support)
- [vic4key](https://github.com/vic4key) : For supporting/contributing on this project
- and [all developers](https://github.com/hacksider/Deep-Live-Cam/graphs/contributors) behind libraries used in this project.
- Foot Note: [This is originally roop-cam, see the full history of the code here.](https://github.com/hacksider/roop-cam) Please be informed that the base author of the code is [s0md3v](https://github.com/s0md3v/roop)

View File

@ -2,19 +2,38 @@ from typing import Any
import cv2
import modules.globals # Import the globals to check the color correction toggle
def list_available_cameras(max_tested: int = 10):
""" List all available camera indices. """
available_cameras = []
for i in range(max_tested):
cap = cv2.VideoCapture(i)
if cap.isOpened():
available_cameras.append(i)
cap.release()
return available_cameras
def get_video_frame(video_path: str, frame_number: int = 0) -> Any:
capture = cv2.VideoCapture(video_path)
def get_video_frame(video_source: Any, frame_number: int = 0, is_camera: bool = False) -> Any:
"""
Capture a video frame from a camera or video file.
:param video_source: The camera index or video file path.
:param frame_number: Frame number to retrieve (only applicable for video files).
:param is_camera: Flag to indicate if the source is a camera.
:return: The captured frame.
"""
capture = cv2.VideoCapture(video_source)
# Set MJPEG format to ensure correct color space handling
capture.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*'MJPG'))
# Only force RGB conversion if color correction is enabled
if modules.globals.color_correction:
capture.set(cv2.CAP_PROP_CONVERT_RGB, 1)
frame_total = capture.get(cv2.CAP_PROP_FRAME_COUNT)
capture.set(cv2.CAP_PROP_POS_FRAMES, min(frame_total, frame_number - 1))
if not is_camera:
frame_total = capture.get(cv2.CAP_PROP_FRAME_COUNT)
capture.set(cv2.CAP_PROP_POS_FRAMES, min(frame_total, frame_number - 1))
has_frame, frame = capture.read()
if has_frame and modules.globals.color_correction:
@ -24,8 +43,8 @@ def get_video_frame(video_path: str, frame_number: int = 0) -> Any:
capture.release()
return frame if has_frame else None
def get_video_frame_total(video_path: str) -> int:
""" Get total frame count of a video file. """
capture = cv2.VideoCapture(video_path)
video_frame_total = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))
capture.release()

View File

@ -0,0 +1,32 @@
import numpy as np
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
from typing import Any
def find_cluster_centroids(embeddings, max_k=10) -> Any:
inertia = []
cluster_centroids = []
K = range(1, max_k+1)
for k in K:
kmeans = KMeans(n_clusters=k, random_state=0)
kmeans.fit(embeddings)
inertia.append(kmeans.inertia_)
cluster_centroids.append({"k": k, "centroids": kmeans.cluster_centers_})
diffs = [inertia[i] - inertia[i+1] for i in range(len(inertia)-1)]
optimal_centroids = cluster_centroids[diffs.index(max(diffs)) + 1]['centroids']
return optimal_centroids
def find_closest_centroid(centroids: list, normed_face_embedding) -> list:
try:
centroids = np.array(centroids)
normed_face_embedding = np.array(normed_face_embedding)
similarities = np.dot(centroids, normed_face_embedding)
closest_centroid_index = np.argmax(similarities)
return closest_centroid_index, centroids[closest_centroid_index]
except ValueError:
return None

View File

@ -40,6 +40,7 @@ def parse_args() -> None:
program.add_argument('--keep-frames', help='keep temporary frames', dest='keep_frames', action='store_true', default=False)
program.add_argument('--many-faces', help='process every face', dest='many_faces', action='store_true', default=False)
program.add_argument('--nsfw-filter', help='filter the NSFW image or video', dest='nsfw_filter', action='store_true', default=False)
program.add_argument('--map-faces', help='map source target faces', dest='map_faces', action='store_true', default=False)
program.add_argument('--video-encoder', help='adjust output video encoder', dest='video_encoder', default='libx264', choices=['libx264', 'libx265', 'libvpx-vp9'])
program.add_argument('--video-quality', help='adjust output video quality', dest='video_quality', type=int, default=18, choices=range(52), metavar='[0-51]')
program.add_argument('--live-mirror', help='The live camera display as you see it in the front-facing camera frame', dest='live_mirror', action='store_true', default=False)
@ -67,6 +68,7 @@ def parse_args() -> None:
modules.globals.keep_frames = args.keep_frames
modules.globals.many_faces = args.many_faces
modules.globals.nsfw_filter = args.nsfw_filter
modules.globals.map_faces = args.map_faces
modules.globals.video_encoder = args.video_encoder
modules.globals.video_quality = args.video_quality
modules.globals.live_mirror = args.live_mirror
@ -194,10 +196,13 @@ def start() -> None:
# process image to videos
if modules.globals.nsfw_filter and ui.check_and_ignore_nsfw(modules.globals.target_path, destroy):
return
update_status('Creating temp resources...')
create_temp(modules.globals.target_path)
update_status('Extracting frames...')
extract_frames(modules.globals.target_path)
if not modules.globals.map_faces:
update_status('Creating temp resources...')
create_temp(modules.globals.target_path)
update_status('Extracting frames...')
extract_frames(modules.globals.target_path)
temp_frame_paths = get_temp_frame_paths(modules.globals.target_path)
for frame_processor in get_frame_processors_modules(modules.globals.frame_processors):
update_status('Progressing...', frame_processor.NAME)

View File

@ -1,8 +1,16 @@
import os
import shutil
from typing import Any
import insightface
import cv2
import numpy as np
import modules.globals
from tqdm import tqdm
from modules.typing import Frame
from modules.cluster_analysis import find_cluster_centroids, find_closest_centroid
from modules.utilities import get_temp_directory_path, create_temp, extract_frames, clean_temp, get_temp_frame_paths
from pathlib import Path
FACE_ANALYSER = None
@ -29,3 +37,153 @@ def get_many_faces(frame: Frame) -> Any:
return get_face_analyser().get(frame)
except IndexError:
return None
def has_valid_map() -> bool:
for map in modules.globals.souce_target_map:
if "source" in map and "target" in map:
return True
return False
def default_source_face() -> Any:
for map in modules.globals.souce_target_map:
if "source" in map:
return map['source']['face']
return None
def simplify_maps() -> Any:
centroids = []
faces = []
for map in modules.globals.souce_target_map:
if "source" in map and "target" in map:
centroids.append(map['target']['face'].normed_embedding)
faces.append(map['source']['face'])
modules.globals.simple_map = {'source_faces': faces, 'target_embeddings': centroids}
return None
def add_blank_map() -> Any:
try:
max_id = -1
if len(modules.globals.souce_target_map) > 0:
max_id = max(modules.globals.souce_target_map, key=lambda x: x['id'])['id']
modules.globals.souce_target_map.append({
'id' : max_id + 1
})
except ValueError:
return None
def get_unique_faces_from_target_image() -> Any:
try:
modules.globals.souce_target_map = []
target_frame = cv2.imread(modules.globals.target_path)
many_faces = get_many_faces(target_frame)
i = 0
for face in many_faces:
x_min, y_min, x_max, y_max = face['bbox']
modules.globals.souce_target_map.append({
'id' : i,
'target' : {
'cv2' : target_frame[int(y_min):int(y_max), int(x_min):int(x_max)],
'face' : face
}
})
i = i + 1
except ValueError:
return None
def get_unique_faces_from_target_video() -> Any:
try:
modules.globals.souce_target_map = []
frame_face_embeddings = []
face_embeddings = []
print('Creating temp resources...')
clean_temp(modules.globals.target_path)
create_temp(modules.globals.target_path)
print('Extracting frames...')
extract_frames(modules.globals.target_path)
temp_frame_paths = get_temp_frame_paths(modules.globals.target_path)
i = 0
for temp_frame_path in tqdm(temp_frame_paths, desc="Extracting face embeddings from frames"):
temp_frame = cv2.imread(temp_frame_path)
many_faces = get_many_faces(temp_frame)
for face in many_faces:
face_embeddings.append(face.normed_embedding)
frame_face_embeddings.append({'frame': i, 'faces': many_faces, 'location': temp_frame_path})
i += 1
centroids = find_cluster_centroids(face_embeddings)
for frame in frame_face_embeddings:
for face in frame['faces']:
closest_centroid_index, _ = find_closest_centroid(centroids, face.normed_embedding)
face['target_centroid'] = closest_centroid_index
for i in range(len(centroids)):
modules.globals.souce_target_map.append({
'id' : i
})
temp = []
for frame in tqdm(frame_face_embeddings, desc=f"Mapping frame embeddings to centroids-{i}"):
temp.append({'frame': frame['frame'], 'faces': [face for face in frame['faces'] if face['target_centroid'] == i], 'location': frame['location']})
modules.globals.souce_target_map[i]['target_faces_in_frame'] = temp
# dump_faces(centroids, frame_face_embeddings)
default_target_face()
except ValueError:
return None
def default_target_face():
for map in modules.globals.souce_target_map:
best_face = None
best_frame = None
for frame in map['target_faces_in_frame']:
if len(frame['faces']) > 0:
best_face = frame['faces'][0]
best_frame = frame
break
for frame in map['target_faces_in_frame']:
for face in frame['faces']:
if face['det_score'] > best_face['det_score']:
best_face = face
best_frame = frame
x_min, y_min, x_max, y_max = best_face['bbox']
target_frame = cv2.imread(best_frame['location'])
map['target'] = {
'cv2' : target_frame[int(y_min):int(y_max), int(x_min):int(x_max)],
'face' : best_face
}
def dump_faces(centroids: Any, frame_face_embeddings: list):
temp_directory_path = get_temp_directory_path(modules.globals.target_path)
for i in range(len(centroids)):
if os.path.exists(temp_directory_path + f"/{i}") and os.path.isdir(temp_directory_path + f"/{i}"):
shutil.rmtree(temp_directory_path + f"/{i}")
Path(temp_directory_path + f"/{i}").mkdir(parents=True, exist_ok=True)
for frame in tqdm(frame_face_embeddings, desc=f"Copying faces to temp/./{i}"):
temp_frame = cv2.imread(frame['location'])
j = 0
for face in frame['faces']:
if face['target_centroid'] == i:
x_min, y_min, x_max, y_max = face['bbox']
if temp_frame[int(y_min):int(y_max), int(x_min):int(x_max)].size > 0:
cv2.imwrite(temp_directory_path + f"/{i}/{frame['frame']}_{j}.png", temp_frame[int(y_min):int(y_max), int(x_min):int(x_max)])
j += 1

View File

@ -1,5 +1,5 @@
import os
from typing import List, Dict
from typing import List, Dict, Any
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
WORKFLOW_DIR = os.path.join(ROOT_DIR, 'workflow')
@ -9,6 +9,9 @@ file_types = [
('Video', ('*.mp4','*.mkv'))
]
souce_target_map = []
simple_map = {}
source_path = None
target_path = None
output_path = None
@ -17,6 +20,7 @@ keep_fps = None
keep_audio = None
keep_frames = None
many_faces = None
map_faces = None
color_correction = None # New global variable for color correction toggle
nsfw_filter = None
video_encoder = None

View File

@ -6,9 +6,10 @@ import threading
import modules.globals
import modules.processors.frame.core
from modules.core import update_status
from modules.face_analyser import get_one_face, get_many_faces
from modules.face_analyser import get_one_face, get_many_faces, default_source_face
from modules.typing import Face, Frame
from modules.utilities import conditional_download, resolve_relative_path, is_image, is_video
from modules.cluster_analysis import find_closest_centroid
FACE_SWAPPER = None
THREAD_LOCK = threading.Lock()
@ -22,10 +23,10 @@ def pre_check() -> bool:
def pre_start() -> bool:
if not is_image(modules.globals.source_path):
if not modules.globals.map_faces and not is_image(modules.globals.source_path):
update_status('Select an image for source path.', NAME)
return False
elif not get_one_face(cv2.imread(modules.globals.source_path)):
elif not modules.globals.map_faces and not get_one_face(cv2.imread(modules.globals.source_path)):
update_status('No face in source path detected.', NAME)
return False
if not is_image(modules.globals.target_path) and not is_video(modules.globals.target_path):
@ -65,26 +66,98 @@ def process_frame(source_face: Face, temp_frame: Frame) -> Frame:
return temp_frame
def process_frame_v2(temp_frame: Frame, temp_frame_path: str = "") -> Frame:
if is_image(modules.globals.target_path):
if modules.globals.many_faces:
source_face = default_source_face()
for map in modules.globals.souce_target_map:
target_face = map['target']['face']
temp_frame = swap_face(source_face, target_face, temp_frame)
elif not modules.globals.many_faces:
for map in modules.globals.souce_target_map:
if "source" in map:
source_face = map['source']['face']
target_face = map['target']['face']
temp_frame = swap_face(source_face, target_face, temp_frame)
elif is_video(modules.globals.target_path):
if modules.globals.many_faces:
source_face = default_source_face()
for map in modules.globals.souce_target_map:
target_frame = [f for f in map['target_faces_in_frame'] if f['location'] == temp_frame_path]
for frame in target_frame:
for target_face in frame['faces']:
temp_frame = swap_face(source_face, target_face, temp_frame)
elif not modules.globals.many_faces:
for map in modules.globals.souce_target_map:
if "source" in map:
target_frame = [f for f in map['target_faces_in_frame'] if f['location'] == temp_frame_path]
source_face = map['source']['face']
for frame in target_frame:
for target_face in frame['faces']:
temp_frame = swap_face(source_face, target_face, temp_frame)
else:
many_faces = get_many_faces(temp_frame)
if modules.globals.many_faces:
source_face = default_source_face()
if many_faces:
for target_face in many_faces:
temp_frame = swap_face(source_face, target_face, temp_frame)
elif not modules.globals.many_faces:
if many_faces:
for target_face in many_faces:
closest_centroid_index, _ = find_closest_centroid(modules.globals.simple_map['target_embeddings'], target_face.normed_embedding)
temp_frame = swap_face(modules.globals.simple_map['source_faces'][closest_centroid_index], target_face, temp_frame)
return temp_frame
def process_frames(source_path: str, temp_frame_paths: List[str], progress: Any = None) -> None:
source_face = get_one_face(cv2.imread(source_path))
for temp_frame_path in temp_frame_paths:
temp_frame = cv2.imread(temp_frame_path)
try:
result = process_frame(source_face, temp_frame)
cv2.imwrite(temp_frame_path, result)
except Exception as exception:
print(exception)
pass
if progress:
progress.update(1)
if not modules.globals.map_faces:
source_face = get_one_face(cv2.imread(source_path))
for temp_frame_path in temp_frame_paths:
temp_frame = cv2.imread(temp_frame_path)
try:
result = process_frame(source_face, temp_frame)
cv2.imwrite(temp_frame_path, result)
except Exception as exception:
print(exception)
pass
if progress:
progress.update(1)
else:
for temp_frame_path in temp_frame_paths:
temp_frame = cv2.imread(temp_frame_path)
try:
result = process_frame_v2(temp_frame, temp_frame_path)
cv2.imwrite(temp_frame_path, result)
except Exception as exception:
print(exception)
pass
if progress:
progress.update(1)
def process_image(source_path: str, target_path: str, output_path: str) -> None:
source_face = get_one_face(cv2.imread(source_path))
target_frame = cv2.imread(target_path)
result = process_frame(source_face, target_frame)
cv2.imwrite(output_path, result)
if not modules.globals.map_faces:
source_face = get_one_face(cv2.imread(source_path))
target_frame = cv2.imread(target_path)
result = process_frame(source_face, target_frame)
cv2.imwrite(output_path, result)
else:
if modules.globals.many_faces:
update_status('Many faces enabled. Using first source image. Progressing...', NAME)
target_frame = cv2.imread(output_path)
result = process_frame_v2(target_frame)
cv2.imwrite(output_path, result)
def process_video(source_path: str, temp_frame_paths: List[str]) -> None:
if modules.globals.map_faces and modules.globals.many_faces:
update_status('Many faces enabled. Using first source image. Progressing...', NAME)
modules.processors.frame.core.process_video(source_path, temp_frame_paths, process_frames)

View File

@ -42,6 +42,16 @@ def init(start: Callable[[], None], destroy: Callable[[], None]) -> ctk.CTk:
PREVIEW = create_preview(ROOT)
return ROOT
def list_available_cameras(max_tested: int = 10):
""" List all available camera indices. """
available_cameras = []
for i in range(max_tested):
cap = cv2.VideoCapture(i)
if cap.isOpened():
available_cameras.append(i)
cap.release()
return available_cameras
def create_root(start: Callable[[], None], destroy: Callable[[], None]) -> ctk.CTk:
@ -67,11 +77,21 @@ def create_root(start: Callable[[], None], destroy: Callable[[], None]) -> ctk.C
select_face_button.place(relx=0.1, rely=0.4, relwidth=0.3, relheight=0.1)
swap_faces_button = ctk.CTkButton(root, text='', cursor='hand2', command=lambda: swap_faces_paths())
swap_faces_button.place(relx=0.45, rely=0.4, relwidth=0.1, relheight=0.1)
swap_faces_button.place(relx=0.45, relwidth=0.1, relheight=0.1)
select_target_button = ctk.CTkButton(root, text='Select a target', cursor='hand2', command=lambda: select_target_path())
select_target_button.place(relx=0.6, rely=0.4, relwidth=0.3, relheight=0.1)
# Create dropdown for camera selection
available_cameras = list_available_cameras()
if available_cameras:
camera_var = ctk.StringVar(value=str(available_cameras[0])) # Default to the first available camera
camera_dropdown = ctk.CTkOptionMenu(root, variable=camera_var, values=[str(i) for i in available_cameras])
camera_dropdown.place(relx=0.1, rely=0.55, relwidth=0.5, relheight=0.05)
else:
camera_var = ctk.StringVar(value="No cameras found")
camera_dropdown = ctk.CTkLabel(root, text="No cameras found")
camera_dropdown.place(relx=0.1, rely=0.55, relwidth=0.5, relheight=0.05)
keep_fps_value = ctk.BooleanVar(value=modules.globals.keep_fps)
keep_fps_checkbox = ctk.CTkSwitch(root, text='Keep fps', variable=keep_fps_value, cursor='hand2', command=lambda: setattr(modules.globals, 'keep_fps', not modules.globals.keep_fps))
keep_fps_checkbox.place(relx=0.1, rely=0.6)
@ -101,6 +121,7 @@ def create_root(start: Callable[[], None], destroy: Callable[[], None]) -> ctk.C
# nsfw_value = ctk.BooleanVar(value=modules.globals.nsfw_filter)
# nsfw_switch = ctk.CTkSwitch(root, text='NSFW filter', variable=nsfw_value, cursor='hand2', command=lambda: setattr(modules.globals, 'nsfw_filter', nsfw_value.get()))
# nsfw_switch.place(relx=0.6, rely=0.7)
start_button = ctk.CTkButton(root, text='Start', cursor='hand2', command=lambda: select_output_path(start))
start_button.place(relx=0.15, rely=0.80, relwidth=0.2, relheight=0.05)
@ -111,7 +132,7 @@ def create_root(start: Callable[[], None], destroy: Callable[[], None]) -> ctk.C
preview_button = ctk.CTkButton(root, text='Preview', cursor='hand2', command=lambda: toggle_preview())
preview_button.place(relx=0.65, rely=0.80, relwidth=0.2, relheight=0.05)
live_button = ctk.CTkButton(root, text='Live', cursor='hand2', command=lambda: webcam_preview())
live_button = ctk.CTkButton(root, text='Live', cursor='hand2', command=lambda: webcam_preview(int(camera_var.get())))
live_button.place(relx=0.40, rely=0.86, relwidth=0.2, relheight=0.05)
status_label = ctk.CTkLabel(root, text=None, justify='center')
@ -125,6 +146,8 @@ def create_root(start: Callable[[], None], destroy: Callable[[], None]) -> ctk.C
return root
def create_preview(parent: ctk.CTkToplevel) -> ctk.CTkToplevel:
global preview_label, preview_slider
@ -315,14 +338,14 @@ def update_preview(frame_number: int = 0) -> None:
update_status('Processing succeed!')
PREVIEW.deiconify()
def webcam_preview():
def webcam_preview(camera_index=0):
if modules.globals.source_path is None:
# No image selected
return
global preview_label, PREVIEW
camera = cv2.VideoCapture(0) # Use index for the webcam (adjust the index accordingly if necessary)
camera = cv2.VideoCapture(camera_index) # Use the selected camera index
camera.set(cv2.CAP_PROP_FRAME_WIDTH, PREVIEW_DEFAULT_WIDTH) # Set the width of the resolution
camera.set(cv2.CAP_PROP_FRAME_HEIGHT, PREVIEW_DEFAULT_HEIGHT) # Set the height of the resolution
camera.set(cv2.CAP_PROP_FPS, 60) # Set the frame rate of the webcam
@ -344,10 +367,10 @@ def webcam_preview():
if source_image is None and modules.globals.source_path:
source_image = get_one_face(cv2.imread(modules.globals.source_path))
temp_frame = frame.copy() #Create a copy of the frame
temp_frame = frame.copy() # Create a copy of the frame
if modules.globals.live_mirror:
temp_frame = cv2.flip(temp_frame, 1) # horizontal flipping
temp_frame = cv2.flip(temp_frame, 1) # Horizontal flipping
if modules.globals.live_resizable:
temp_frame = fit_image_to_size(temp_frame, PREVIEW.winfo_width(), PREVIEW.winfo_height())