feat: Implement Optical Flow KPS tracking for webcam performance
Introduces Nth-frame full face detection combined with KCF bounding box tracking and Lucas-Kanade (LK) optical flow for keypoint (KPS) tracking on intermediate frames. This is primarily for single-face webcam mode to improve performance while maintaining per-frame swaps. Key Changes: - Modified `face_swapper.py` (`process_frame`): - Full `insightface.FaceAnalysis` runs every N frames (default 5) or if tracking is lost. - KCF tracker updates bounding box on intermediate frames. - Optical flow (`cv2.calcOpticalFlowPyrLK`) tracks the 5 keypoints from the previous frame to the current intermediate frame. - A `Face` object is constructed with tracked bbox and KPS for swapping on intermediate frames (detailed landmarks like `landmark_2d_106` are None for these). - Experimental similar logic added to `_process_live_target_v2` for `map_faces=True` live mode (non-many_faces path). - Robustness: - Mouth masking and face mask creation functions in `face_swapper.py` now handle cases where `landmark_2d_106` is `None` (e.g., by skipping mouth mask or using bbox for face mask). - Added division-by-zero check in `apply_color_transfer`. - State Management: - Introduced `reset_tracker_state()` in `face_swapper.py` to clear all tracking-related global variables. - `ui.py` now calls `reset_tracker_state()` at appropriate points (webcam start, mode changes, new source image selection) to ensure clean tracking for new sessions. - `DETECTION_INTERVAL` in `face_swapper.py` increased to 5. This aims to provide you with a smoother face swap experience with better FPS by reducing the frequency of expensive full face analysis, while the actual swap operation continues on every frame using tracked data.pull/1298/head
parent
a01314b52c
commit
4e36622a47
File diff suppressed because it is too large
Load Diff
|
@ -19,6 +19,7 @@ from modules.face_analyser import (
|
|||
)
|
||||
from modules.capturer import get_video_frame, get_video_frame_total
|
||||
from modules.processors.frame.core import get_frame_processors_modules
|
||||
from modules.processors.frame.face_swapper import reset_tracker_state # Added import
|
||||
from modules.utilities import (
|
||||
is_image,
|
||||
is_video,
|
||||
|
@ -240,6 +241,7 @@ def create_root(start: Callable[[], None], destroy: Callable[[], None]) -> ctk.C
|
|||
command=lambda: (
|
||||
setattr(modules.globals, "many_faces", many_faces_value.get()),
|
||||
save_switch_states(),
|
||||
reset_tracker_state() # Added reset call
|
||||
),
|
||||
)
|
||||
many_faces_switch.place(relx=0.6, rely=0.65)
|
||||
|
@ -266,7 +268,8 @@ def create_root(start: Callable[[], None], destroy: Callable[[], None]) -> ctk.C
|
|||
command=lambda: (
|
||||
setattr(modules.globals, "map_faces", map_faces.get()),
|
||||
save_switch_states(),
|
||||
close_mapper_window() if not map_faces.get() else None
|
||||
close_mapper_window() if not map_faces.get() else None,
|
||||
reset_tracker_state() # Added reset call
|
||||
),
|
||||
)
|
||||
map_faces_switch.place(relx=0.1, rely=0.75)
|
||||
|
@ -604,9 +607,11 @@ def select_source_path() -> None:
|
|||
RECENT_DIRECTORY_SOURCE = os.path.dirname(modules.globals.source_path)
|
||||
image = render_image_preview(modules.globals.source_path, (200, 200))
|
||||
source_label.configure(image=image)
|
||||
reset_tracker_state() # Added reset call
|
||||
else:
|
||||
modules.globals.source_path = None
|
||||
source_label.configure(image=None)
|
||||
reset_tracker_state() # Added reset call even if source is cleared
|
||||
|
||||
|
||||
def swap_faces_paths() -> None:
|
||||
|
@ -979,6 +984,8 @@ def create_webcam_preview(camera_index: int):
|
|||
frame_count = 0
|
||||
fps = 0
|
||||
|
||||
reset_tracker_state() # Ensure tracker is reset before starting webcam loop
|
||||
|
||||
while True:
|
||||
ret, frame = cap.read()
|
||||
if not ret:
|
||||
|
|
Loading…
Reference in New Issue