mirrors/facefusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #812 from facefusion/improvement/deep-swapper-alignment

Browse Source 
Improvement/deep swapper alignment
This commit is contained in:
Harisreedhar 2024-11-13 19:50:30 +05:30 committed by GitHub
commit c7e7751b81
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 82 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
facefusion/face_helper.py
View File

@ -5,7 +5,7 @@ import cv2
import numpy
from cv2.typing import Size
from facefusion.typing import Anchors, Angle, BoundingBox, Distance, FaceDetectorModel, FaceLandmark5, FaceLandmark68, Mask, Matrix, Points, Scale, Score, Translation, VisionFrame, WarpTemplate, WarpTemplateSet
from facefusion.typing import Anchors, Angle, BoundingBox, Direction, Distance, FaceDetectorModel, FaceLandmark5, FaceLandmark68, Mask, Matrix, Points, PointsTemplate, PointsTemplateSet, Scale, Score, Translation, VisionFrame, WarpTemplate, WarpTemplateSet
WARP_TEMPLATES : WarpTemplateSet =\
{
@ -16,7 +16,7 @@ WARP_TEMPLATES : WarpTemplateSet =\
[ 0.50000000, 0.61154464 ],
[ 0.37913393, 0.77687500 ],
[ 0.62086607, 0.77687500 ]
]),
]).astype(numpy.float32),
'arcface_112_v2': numpy.array(
[
[ 0.34191607, 0.46157411 ],
@ -24,7 +24,7 @@ WARP_TEMPLATES : WarpTemplateSet =\
[ 0.50022500, 0.64050536 ],
[ 0.37097589, 0.82469196 ],
[ 0.63151696, 0.82325089 ]
]),
]).astype(numpy.float32),
'arcface_128_v2': numpy.array(
[
[ 0.36167656, 0.40387734 ],
@ -32,7 +32,15 @@ WARP_TEMPLATES : WarpTemplateSet =\
[ 0.50019687, 0.56044219 ],
[ 0.38710391, 0.72160547 ],
[ 0.61507734, 0.72034453 ]
]),
]).astype(numpy.float32),
'deep_face_live': numpy.array(
[
[ 0.22549182, 0.21599032 ],
[ 0.75476142, 0.21599032 ],
[ 0.49012712, 0.51562511 ],
[ 0.25414925, 0.78023333 ],
[ 0.72610437, 0.78023333 ]
]).astype(numpy.float32),
'ffhq_512': numpy.array(
[
[ 0.37691676, 0.46864664 ],
@ -40,7 +48,7 @@ WARP_TEMPLATES : WarpTemplateSet =\
[ 0.50123859, 0.61331904 ],
[ 0.39308822, 0.72541100 ],
[ 0.61150205, 0.72490465 ]
]),
]).astype(numpy.float32),
'mtcnn_512': numpy.array(
[
[ 0.36562865, 0.46733799 ],
@ -48,7 +56,7 @@ WARP_TEMPLATES : WarpTemplateSet =\
[ 0.50019127, 0.61942959 ],
[ 0.39032951, 0.77598822 ],
[ 0.61178945, 0.77476328 ]
]),
]).astype(numpy.float32),
'styleganex_384': numpy.array(
[
[ 0.42353745, 0.52289879 ],
@ -56,7 +64,29 @@ WARP_TEMPLATES : WarpTemplateSet =\
[ 0.50123859, 0.61331904 ],
[ 0.43364461, 0.68337652 ],
[ 0.57015325, 0.68306005 ]
])
]).astype(numpy.float32)
}
POINTS_TEMPLATES : PointsTemplateSet =\
{
'square': numpy.array(
[
[ 0, 0 ],
[ 1, 0 ],
[ 1, 1 ],
[ 0, 1 ]
]).astype(numpy.float32),
'triangle_orthogonal': numpy.array(
[
[ 0, 0 ],
[ 1, 0 ],
[ 0, 1 ]
]).astype(numpy.float32),
'triangle_skew': numpy.array(
[
[ 0, 0 ],
[ 1, 0 ],
[ 1, 1 ]
]).astype(numpy.float32)
}
@ -66,16 +96,37 @@ def estimate_matrix_by_face_landmark_5(face_landmark_5 : FaceLandmark5, warp_tem
return affine_matrix
def estimate_matrix_by_points(source_points : Points, polygon_template : PointsTemplate, crop_size : Size) -> Matrix:
target_points = POINTS_TEMPLATES.get(polygon_template) * crop_size
affine_matrix = cv2.getAffineTransform(source_points, target_points.astype(numpy.float32))
return affine_matrix
def warp_face_by_face_landmark_5(temp_vision_frame : VisionFrame, face_landmark_5 : FaceLandmark5, warp_template : WarpTemplate, crop_size : Size) -> Tuple[VisionFrame, Matrix]:
affine_matrix = estimate_matrix_by_face_landmark_5(face_landmark_5, warp_template, crop_size)
crop_vision_frame = cv2.warpAffine(temp_vision_frame, affine_matrix, crop_size, borderMode = cv2.BORDER_REPLICATE, flags = cv2.INTER_AREA)
return crop_vision_frame, affine_matrix
def warp_face_for_deepfacelive(temp_vision_frame : VisionFrame, face_landmark_5 : FaceLandmark5, crop_size : Size, shift : Tuple[float, float], coverage : float) -> Tuple[VisionFrame, Matrix]:
affine_matrix = estimate_matrix_by_face_landmark_5(face_landmark_5, 'deep_face_live', (1, 1))
square_points = POINTS_TEMPLATES.get('square')
square_points = transform_points(square_points, cv2.invertAffineTransform(affine_matrix))
center_point = square_points.mean(axis = 0)
center_point += shift[0] * numpy.subtract(square_points[1], square_points[0])
center_point += shift[1] * numpy.subtract(square_points[3], square_points[0])
scale = numpy.linalg.norm(center_point - square_points[0]) * coverage
top_bottom_direction = calc_points_direction(square_points[0], square_points[2]) * scale
bottom_top_direction = calc_points_direction(square_points[3], square_points[1]) * scale
source_points = numpy.array([ center_point - top_bottom_direction, center_point + bottom_top_direction, center_point + top_bottom_direction ]).astype(numpy.float32)
affine_matrix = estimate_matrix_by_points(source_points, 'triangle_skew', crop_size)
crop_vision_frame = cv2.warpAffine(temp_vision_frame, affine_matrix, crop_size, flags = cv2.INTER_CUBIC)
return crop_vision_frame, affine_matrix
def warp_face_by_bounding_box(temp_vision_frame : VisionFrame, bounding_box : BoundingBox, crop_size : Size) -> Tuple[VisionFrame, Matrix]:
source_points = numpy.array([ [ bounding_box[0], bounding_box[1] ], [bounding_box[2], bounding_box[1] ], [ bounding_box[0], bounding_box[3] ] ]).astype(numpy.float32)
target_points = numpy.array([ [ 0, 0 ], [ crop_size[0], 0 ], [ 0, crop_size[1] ] ]).astype(numpy.float32)
affine_matrix = cv2.getAffineTransform(source_points, target_points)
affine_matrix = estimate_matrix_by_points(source_points, 'triangle_orthogonal', crop_size)
if bounding_box[2] - bounding_box[0] > crop_size[0] or bounding_box[3] - bounding_box[1] > crop_size[1]:
interpolation_method = cv2.INTER_AREA
else:
@ -102,6 +153,12 @@ def paste_back(temp_vision_frame : VisionFrame, crop_vision_frame : VisionFrame,
return paste_vision_frame
def calc_points_direction(start_point : Points, end_point : Points) -> Direction:
direction = end_point - start_point
direction /= numpy.linalg.norm(direction)
return direction
@lru_cache(maxsize = None)
def create_static_anchors(feature_stride : int, anchor_total : int, stride_height : int, stride_width : int) -> Anchors:
y, x = numpy.mgrid[:stride_height, :stride_width][::-1]

20
facefusion/processors/modules/deep_swapper.py
View File

@ -10,7 +10,7 @@ import facefusion.processors.core as processors
from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, wording
from facefusion.download import conditional_download_hashes, conditional_download_sources
from facefusion.face_analyser import get_many_faces, get_one_face
from facefusion.face_helper import paste_back, warp_face_by_face_landmark_5
from facefusion.face_helper import paste_back, warp_face_for_deepfacelive
from facefusion.face_masker import create_occlusion_mask, create_static_box_mask
from facefusion.face_selector import find_similar_faces, sort_and_filter_faces
from facefusion.face_store import get_reference_faces
@ -42,8 +42,9 @@ MODEL_SET : ModelSet =\
'path': resolve_relative_path('../.assets/models/Jackie_Chan.dfm')
}
},
'template': 'arcface_128_v2',
'size': (224, 224)
'size': (224, 224),
'shift': (0.0, 0.0),
'coverage': 2.2
}
}
@ -110,9 +111,10 @@ def post_process() -> None:
def swap_face(target_face : Face, temp_vision_frame : VisionFrame) -> VisionFrame:
model_template = get_model_options().get('template')
model_size = get_model_options().get('size')
crop_vision_frame, affine_matrix = warp_face_by_face_landmark_5(temp_vision_frame, target_face.landmark_set.get('5/68'), model_template, model_size)
model_shift = get_model_options().get('shift')
model_coverage = get_model_options().get('coverage')
crop_vision_frame, affine_matrix = warp_face_for_deepfacelive(temp_vision_frame, target_face.landmark_set.get('5/68'), model_size, model_shift, model_coverage)
crop_vision_frame_raw = crop_vision_frame.copy()
box_mask = create_static_box_mask(crop_vision_frame.shape[:2][::-1], state_manager.get_item('face_mask_blur'), state_manager.get_item('face_mask_padding'))
crop_masks =\
@ -166,10 +168,12 @@ def normalize_crop_frame(crop_vision_frame : VisionFrame) -> VisionFrame:
def prepare_crop_mask(crop_source_mask : Mask, crop_target_mask : Mask) -> Mask:
model_size = get_model_options().get('size')
crop_mask = numpy.maximum.reduce([ crop_source_mask, crop_target_mask ])
blur_size = 6.25
kernel_size = 3
crop_mask = numpy.minimum.reduce([ crop_source_mask, crop_target_mask ])
crop_mask = crop_mask.reshape(model_size).clip(0, 1)
crop_mask = cv2.erode(crop_mask, numpy.ones((5, 5), numpy.uint8), iterations = 1)
crop_mask = cv2.GaussianBlur(crop_mask, (9, 9), 0)
crop_mask = cv2.erode(crop_mask, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel_size, kernel_size)), iterations = 2)
crop_mask = cv2.GaussianBlur(crop_mask, (0, 0), blur_size)
return crop_mask

5
facefusion/typing.py
View File

@ -53,6 +53,7 @@ FaceStore = TypedDict('FaceStore',
VisionFrame = NDArray[Any]
Mask = NDArray[Any]
Points = NDArray[Any]
Direction = NDArray[Any]
Distance = NDArray[Any]
Matrix = NDArray[Any]
Anchors = NDArray[Any]
@ -85,8 +86,10 @@ ProcessStep = Callable[[str, int, Args], bool]
Content = Dict[str, Any]
WarpTemplate = Literal['arcface_112_v1', 'arcface_112_v2', 'arcface_128_v2', 'ffhq_512', 'mtcnn_512', 'styleganex_384']
WarpTemplate = Literal['arcface_112_v1', 'arcface_112_v2', 'arcface_128_v2', 'deep_face_live', 'ffhq_512', 'mtcnn_512', 'styleganex_384']
WarpTemplateSet = Dict[WarpTemplate, NDArray[Any]]
PointsTemplate = Literal['square', 'triangle_orthogonal', 'triangle_skew']
PointsTemplateSet = Dict[PointsTemplate, NDArray[Any]]
ProcessMode = Literal['output', 'preview', 'stream']
ErrorCode = Literal[0, 1, 2, 3, 4]