Merge 0a3430b791 into aa94f2ae7e

@refer to the PR https://github.com/hacksider/Deep-Live-Cam/pull/293 in the `experimental` branch.

README.md

@@ -107,7 +107,7 @@ pip install onnxruntime-directml==1.15.1
 2.  Usage in case the provider is available:
 
 ```
-python run.py --execution-provider directml
+python run.py --execution-provider dml
 
 ```
 
@@ -152,16 +152,19 @@ Additional command line arguments are given below. To learn out what they do, ch
 ```
 options:
   -h, --help                                               show this help message and exit
-  -s SOURCE_PATH, --source SOURCE_PATH                     select an source image
+  -s SOURCE_PATH, --source SOURCE_PATH                     select a source image
-  -t TARGET_PATH, --target TARGET_PATH                     select an target image or video
+  -t TARGET_PATH, --target TARGET_PATH                     select a target image or video
   -o OUTPUT_PATH, --output OUTPUT_PATH                     select output file or directory
   --frame-processor FRAME_PROCESSOR [FRAME_PROCESSOR ...]  frame processors (choices: face_swapper, face_enhancer, ...)
   --keep-fps                                               keep original fps
   --keep-audio                                             keep original audio
   --keep-frames                                            keep temporary frames
   --many-faces                                             process every face
+  --nsfw-filter                                            filter the NSFW image or video
   --video-encoder {libx264,libx265,libvpx-vp9}             adjust output video encoder
   --video-quality [0-51]                                   adjust output video quality
+  --live-mirror                                            the live camera display as you see it in the front-facing camera frame
+  --live-resizable                                         the live camera frame is resizable
   --max-memory MAX_MEMORY                                  maximum amount of RAM in GB
   --execution-provider {cpu} [{cpu} ...]                   available execution provider (choices: cpu, ...)
   --execution-threads EXECUTION_THREADS                    number of execution threads

modules/core.py

@@ -39,8 +39,11 @@ def parse_args() -> None:
     program.add_argument('--keep-audio', help='keep original audio', dest='keep_audio', action='store_true', default=True)
     program.add_argument('--keep-frames', help='keep temporary frames', dest='keep_frames', action='store_true', default=False)
     program.add_argument('--many-faces', help='process every face', dest='many_faces', action='store_true', default=False)
+    program.add_argument('--nsfw-filter', help='filter the NSFW image or video', dest='nsfw_filter', action='store_true', default=False)
     program.add_argument('--video-encoder', help='adjust output video encoder', dest='video_encoder', default='libx264', choices=['libx264', 'libx265', 'libvpx-vp9'])
     program.add_argument('--video-quality', help='adjust output video quality', dest='video_quality', type=int, default=18, choices=range(52), metavar='[0-51]')
+    program.add_argument('--live-mirror', help='The live camera display as you see it in the front-facing camera frame', dest='live_mirror', action='store_true', default=False)
+    program.add_argument('--live-resizable', help='The live camera frame is resizable', dest='live_resizable', action='store_true', default=False)
     program.add_argument('--max-memory', help='maximum amount of RAM in GB', dest='max_memory', type=int, default=suggest_max_memory())
     program.add_argument('--execution-provider', help='execution provider', dest='execution_provider', default=['cpu'], choices=suggest_execution_providers(), nargs='+')
     program.add_argument('--execution-threads', help='number of execution threads', dest='execution_threads', type=int, default=suggest_execution_threads())
@@ -63,8 +66,11 @@ def parse_args() -> None:
     modules.globals.keep_audio = args.keep_audio
     modules.globals.keep_frames = args.keep_frames
     modules.globals.many_faces = args.many_faces
+    modules.globals.nsfw_filter = args.nsfw_filter
     modules.globals.video_encoder = args.video_encoder
     modules.globals.video_quality = args.video_quality
+    modules.globals.live_mirror = args.live_mirror
+    modules.globals.live_resizable = args.live_resizable
     modules.globals.max_memory = args.max_memory
     modules.globals.execution_providers = decode_execution_providers(args.execution_provider)
     modules.globals.execution_threads = args.execution_threads
@@ -75,8 +81,6 @@ def parse_args() -> None:
     else:
         modules.globals.fp_ui['face_enhancer'] = False
 
-    modules.globals.nsfw = False
-
     # translate deprecated args
     if args.source_path_deprecated:
         print('\033[33mArgument -f and --face are deprecated. Use -s and --source instead.\033[0m')
@@ -169,13 +173,15 @@ def start() -> None:
     for frame_processor in get_frame_processors_modules(modules.globals.frame_processors):
         if not frame_processor.pre_start():
             return
+    update_status('Processing...')
     # process image to image
     if has_image_extension(modules.globals.target_path):
-        if modules.globals.nsfw == False:
+        if modules.globals.nsfw_filter and ui.check_and_ignore_nsfw(modules.globals.target_path, destroy):
-            from modules.predicter import predict_image
+            return
-            if predict_image(modules.globals.target_path):
+        try:
-                destroy()
             shutil.copy2(modules.globals.target_path, modules.globals.output_path)
+        except Exception as e:
+            print("Error copying file:", str(e))
         for frame_processor in get_frame_processors_modules(modules.globals.frame_processors):
             update_status('Progressing...', frame_processor.NAME)
             frame_processor.process_image(modules.globals.source_path, modules.globals.output_path, modules.globals.output_path)
@@ -186,10 +192,8 @@ def start() -> None:
             update_status('Processing to image failed!')
         return
     # process image to videos
-    if modules.globals.nsfw == False:
+    if modules.globals.nsfw_filter and ui.check_and_ignore_nsfw(modules.globals.target_path, destroy):
-        from modules.predicter import predict_video
+        return
-        if predict_video(modules.globals.target_path):
-            destroy()
     update_status('Creating temp resources...')
     create_temp(modules.globals.target_path)
     update_status('Extracting frames...')
@@ -225,10 +229,10 @@ def start() -> None:
         update_status('Processing to video failed!')
 
 
-def destroy() -> None:
+def destroy(to_quit=True) -> None:
     if modules.globals.target_path:
         clean_temp(modules.globals.target_path)
-    quit()
+    if to_quit: quit()
 
 
 def run() -> None:

modules/globals.py

@@ -17,14 +17,16 @@ keep_fps = None
 keep_audio = None
 keep_frames = None
 many_faces = None
+nsfw_filter = None
 video_encoder = None
 video_quality = None
+live_mirror = None
+live_resizable = None
 max_memory = None
 execution_providers: List[str] = []
 execution_threads = None
 headless = None
 log_level = 'error'
 fp_ui: Dict[str, bool] = {}
-nsfw = None
 camera_input_combobox = None
 webcam_preview_running = False

modules/predicter.py

@@ -6,11 +6,14 @@ from modules.typing import Frame
 
 MAX_PROBABILITY = 0.85
 
+# Preload the model once for efficiency
+model = None
 
 def predict_frame(target_frame: Frame) -> bool:
     image = Image.fromarray(target_frame)
     image = opennsfw2.preprocess_image(image, opennsfw2.Preprocessing.YAHOO)
-    model = opennsfw2.make_open_nsfw_model()
+    global model
+    if model is None: model = opennsfw2.make_open_nsfw_model()
     views = numpy.expand_dims(image, axis=0)
     _, probability = model.predict(views)[0]
     return probability > MAX_PROBABILITY

modules/ui.py

@@ -10,7 +10,7 @@ import modules.metadata
 from modules.face_analyser import get_one_face
 from modules.capturer import get_video_frame, get_video_frame_total
 from modules.processors.frame.core import get_frame_processors_modules
-from modules.utilities import is_image, is_video, resolve_relative_path
+from modules.utilities import is_image, is_video, resolve_relative_path, has_image_extension
 
 ROOT = None
 ROOT_HEIGHT = 700
@@ -19,6 +19,8 @@ ROOT_WIDTH = 600
 PREVIEW = None
 PREVIEW_MAX_HEIGHT = 700
 PREVIEW_MAX_WIDTH  = 1200
+PREVIEW_DEFAULT_WIDTH  = 960
+PREVIEW_DEFAULT_HEIGHT = 540
 
 RECENT_DIRECTORY_SOURCE = None
 RECENT_DIRECTORY_TARGET = None
@@ -88,9 +90,9 @@ def create_root(start: Callable[[], None], destroy: Callable[[], None]) -> ctk.C
     many_faces_switch = ctk.CTkSwitch(root, text='Many faces', variable=many_faces_value, cursor='hand2', command=lambda: setattr(modules.globals, 'many_faces', many_faces_value.get()))
     many_faces_switch.place(relx=0.6, rely=0.65)
 
-#    nsfw_value = ctk.BooleanVar(value=modules.globals.nsfw)
+    nsfw_value = ctk.BooleanVar(value=modules.globals.nsfw_filter)
-#    nsfw_switch = ctk.CTkSwitch(root, text='NSFW', variable=nsfw_value, cursor='hand2', command=lambda: setattr(modules.globals, 'nsfw', nsfw_value.get()))
+    nsfw_switch = ctk.CTkSwitch(root, text='NSFW filter', variable=nsfw_value, cursor='hand2', command=lambda: setattr(modules.globals, 'nsfw_filter', nsfw_value.get()))
-#    nsfw_switch.place(relx=0.6, rely=0.7)
+    nsfw_switch.place(relx=0.6, rely=0.7)
 
     start_button = ctk.CTkButton(root, text='Start', cursor='hand2', command=lambda: select_output_path(start))
     start_button.place(relx=0.15, rely=0.80, relwidth=0.2, relheight=0.05)
@@ -123,7 +125,7 @@ def create_preview(parent: ctk.CTkToplevel) -> ctk.CTkToplevel:
     preview.title('Preview')
     preview.configure()
     preview.protocol('WM_DELETE_WINDOW', lambda: toggle_preview())
-    preview.resizable(width=False, height=False)
+    preview.resizable(width=True, height=True)
 
     preview_label = ctk.CTkLabel(preview, text=None)
     preview_label.pack(fill='both', expand=True)
@@ -192,6 +194,38 @@ def select_output_path(start: Callable[[], None]) -> None:
         start()
 
 
+def check_and_ignore_nsfw(target, destroy: Callable = None) -> bool:
+    ''' Check if the target is NSFW.
+    TODO: Consider to make blur the target.
+    '''
+    from numpy import ndarray
+    from modules.predicter import predict_image, predict_video, predict_frame
+    if type(target) is str: # image/video file path
+        check_nsfw = predict_image if has_image_extension(target) else predict_video
+    elif type(target) is ndarray: # frame object
+        check_nsfw = predict_frame
+    if check_nsfw and check_nsfw(target):
+        if destroy: destroy(to_quit=False) # Do not need to destroy the window frame if the target is NSFW
+        update_status('Processing ignored!')
+        return True
+    else: return False
+
+
+def fit_image_to_size(image, width: int, height: int):
+    if width is None and height is None:
+      return image
+    h, w, _ = image.shape
+    ratio_h = 0.0
+    ratio_w = 0.0
+    if width > height:
+        ratio_h = height / h
+    else:
+        ratio_w = width  / w
+    ratio = max(ratio_w, ratio_h)
+    new_size = (int(ratio * w), int(ratio * h))
+    return cv2.resize(image, dsize=new_size)
+
+
 def render_image_preview(image_path: str, size: Tuple[int, int]) -> ctk.CTkImage:
     image = Image.open(image_path)
     if size:
@@ -219,7 +253,6 @@ def toggle_preview() -> None:
     elif modules.globals.source_path and modules.globals.target_path:
         init_preview()
         update_preview()
-        PREVIEW.deiconify()
 
 
 def init_preview() -> None:
@@ -234,11 +267,10 @@ def init_preview() -> None:
 
 def update_preview(frame_number: int = 0) -> None:
     if modules.globals.source_path and modules.globals.target_path:
+        update_status('Processing...')
         temp_frame = get_video_frame(modules.globals.target_path, frame_number)
-        if modules.globals.nsfw == False:
+        if modules.globals.nsfw_filter and check_and_ignore_nsfw(temp_frame):
-            from modules.predicter import predict_frame
+            return
-            if predict_frame(temp_frame):
-                quit()
         for frame_processor in get_frame_processors_modules(modules.globals.frame_processors):
             temp_frame = frame_processor.process_frame(
                 get_one_face(cv2.imread(modules.globals.source_path)),
@@ -248,6 +280,8 @@ def update_preview(frame_number: int = 0) -> None:
         image = ImageOps.contain(image, (PREVIEW_MAX_WIDTH, PREVIEW_MAX_HEIGHT), Image.LANCZOS)
         image = ctk.CTkImage(image, size=image.size)
         preview_label.configure(image=image)
+        update_status('Processing succeed!')
+        PREVIEW.deiconify()
 
 def webcam_preview():
     if modules.globals.source_path is None:
@@ -256,14 +290,12 @@ def webcam_preview():
 
     global preview_label, PREVIEW
 
-    cap = cv2.VideoCapture(0)  # Use index for the webcam (adjust the index accordingly if necessary)    
+    camera = cv2.VideoCapture(0)                                    # Use index for the webcam (adjust the index accordingly if necessary)    
-    cap.set(cv2.CAP_PROP_FRAME_WIDTH, 960)  # Set the width of the resolution
+    camera.set(cv2.CAP_PROP_FRAME_WIDTH, PREVIEW_DEFAULT_WIDTH)     # Set the width of the resolution
-    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 540)  # Set the height of the resolution
+    camera.set(cv2.CAP_PROP_FRAME_HEIGHT, PREVIEW_DEFAULT_HEIGHT)   # Set the height of the resolution
-    cap.set(cv2.CAP_PROP_FPS, 60)  # Set the frame rate of the webcam
+    camera.set(cv2.CAP_PROP_FPS, 60)                                # Set the frame rate of the webcam
-    PREVIEW_MAX_WIDTH = 960
-    PREVIEW_MAX_HEIGHT = 540
 
-    preview_label.configure(image=None)  # Reset the preview image before startup
+    preview_label.configure(width=PREVIEW_DEFAULT_WIDTH, height=PREVIEW_DEFAULT_HEIGHT)  # Reset the preview image before startup
 
     PREVIEW.deiconify()  # Open preview window
 
@@ -271,8 +303,8 @@ def webcam_preview():
 
     source_image = None  # Initialize variable for the selected face image
 
-    while True:
+    while camera:
-        ret, frame = cap.read()
+        ret, frame = camera.read()
         if not ret:
             break
 
@@ -282,12 +314,18 @@ def webcam_preview():
 
         temp_frame = frame.copy()  #Create a copy of the frame
 
+        if modules.globals.live_mirror:
+            temp_frame = cv2.flip(temp_frame, 1) # horizontal flipping
+
+        if modules.globals.live_resizable:
+            temp_frame = fit_image_to_size(temp_frame, PREVIEW.winfo_width(), PREVIEW.winfo_height())
+
         for frame_processor in frame_processors:
             temp_frame = frame_processor.process_frame(source_image, temp_frame)
 
         image = cv2.cvtColor(temp_frame, cv2.COLOR_BGR2RGB)  # Convert the image to RGB format to display it with Tkinter
         image = Image.fromarray(image)
-        image = ImageOps.contain(image, (PREVIEW_MAX_WIDTH, PREVIEW_MAX_HEIGHT), Image.LANCZOS)
+        image = ImageOps.contain(image, (temp_frame.shape[1], temp_frame.shape[0]), Image.LANCZOS)
         image = ctk.CTkImage(image, size=image.size)
         preview_label.configure(image=image)
         ROOT.update()
@@ -295,5 +333,5 @@ def webcam_preview():
         if PREVIEW.state() == 'withdrawn':
             break
 
-    cap.release()
+    camera.release()
     PREVIEW.withdraw()  # Close preview window when loop is finished