mirrors/facefusion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #815 from facefusion/changes/dfl-template-approach

Browse Source 
change dfl to template alignment
This commit is contained in:
Harisreedhar 2024-11-15 21:11:04 +05:30 committed by GitHub
commit 28f7dba897
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 26 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
facefusion/face_helper.py
View File

@ -5,7 +5,7 @@ import cv2
import numpy
from cv2.typing import Size
from facefusion.typing import Anchors, Angle, BoundingBox, Direction, Distance, FaceDetectorModel, FaceLandmark5, FaceLandmark68, Mask, Matrix, Points, PointsTemplate, PointsTemplateSet, Scale, Score, Translation, VisionFrame, WarpTemplate, WarpTemplateSet
from facefusion.typing import Anchors, Angle, BoundingBox, Distance, FaceDetectorModel, FaceLandmark5, FaceLandmark68, Mask, Matrix, Points, Scale, Score, Translation, VisionFrame, WarpTemplate, WarpTemplateSet
WARP_TEMPLATES : WarpTemplateSet =\
{
@ -33,13 +33,21 @@ WARP_TEMPLATES : WarpTemplateSet =\
[ 0.38710391, 0.72160547 ],
[ 0.61507734, 0.72034453 ]
]).astype(numpy.float32),
'deepfacelive': numpy.array(
'dfl_head': numpy.array(
[
[ 0.22549182, 0.21599032 ],
[ 0.75476142, 0.21599032 ],
[ 0.49012712, 0.51562511 ],
[ 0.25414925, 0.78023333 ],
[ 0.72610437, 0.78023333 ]
[ 0.39062554, 0.59114583 ],
[ 0.60481776, 0.58951825 ],
[ 0.49414062, 0.70898442 ],
[ 0.41796875, 0.82031255 ],
[ 0.58007813, 0.82031255 ]
]).astype(numpy.float32),
'dfl_whole_face': numpy.array(
[
[ 0.36816406, 0.40429688 ],
[ 0.62565106, 0.40071616 ],
[ 0.49023438, 0.54296875 ],
[ 0.39843755, 0.68554699 ],
[ 0.59375543, 0.68554699 ]
]).astype(numpy.float32),
'ffhq_512': numpy.array(
[
@ -66,28 +74,6 @@ WARP_TEMPLATES : WarpTemplateSet =\
[ 0.57015325, 0.68306005 ]
]).astype(numpy.float32)
}
POINTS_TEMPLATES : PointsTemplateSet =\
{
'square': numpy.array(
[
[ 0, 0 ],
[ 1, 0 ],
[ 1, 1 ],
[ 0, 1 ]
]).astype(numpy.float32),
'triangle_orthogonal': numpy.array(
[
[ 0, 0 ],
[ 1, 0 ],
[ 0, 1 ]
]).astype(numpy.float32),
'triangle_skew': numpy.array(
[
[ 0, 0 ],
[ 1, 0 ],
[ 1, 1 ]
]).astype(numpy.float32)
}
def estimate_matrix_by_face_landmark_5(face_landmark_5 : FaceLandmark5, warp_template : WarpTemplate, crop_size : Size) -> Matrix:
@ -96,37 +82,16 @@ def estimate_matrix_by_face_landmark_5(face_landmark_5 : FaceLandmark5, warp_tem
return affine_matrix
def estimate_matrix_by_points(source_points : Points, polygon_template : PointsTemplate, crop_size : Size) -> Matrix:
target_points = POINTS_TEMPLATES.get(polygon_template) * crop_size
affine_matrix = cv2.getAffineTransform(source_points, target_points.astype(numpy.float32))
return affine_matrix
def warp_face_by_face_landmark_5(temp_vision_frame : VisionFrame, face_landmark_5 : FaceLandmark5, warp_template : WarpTemplate, crop_size : Size) -> Tuple[VisionFrame, Matrix]:
affine_matrix = estimate_matrix_by_face_landmark_5(face_landmark_5, warp_template, crop_size)
crop_vision_frame = cv2.warpAffine(temp_vision_frame, affine_matrix, crop_size, borderMode = cv2.BORDER_REPLICATE, flags = cv2.INTER_AREA)
return crop_vision_frame, affine_matrix
def warp_face_for_deepfacelive(temp_vision_frame : VisionFrame, face_landmark_5 : FaceLandmark5, crop_size : Size, shift : Tuple[float, float], coverage : float) -> Tuple[VisionFrame, Matrix]:
affine_matrix = estimate_matrix_by_face_landmark_5(face_landmark_5, 'deepfacelive', (1, 1))
square_points = POINTS_TEMPLATES.get('square')
square_points = transform_points(square_points, cv2.invertAffineTransform(affine_matrix))
center_point = square_points.mean(axis = 0)
center_point += shift[0] * numpy.subtract(square_points[1], square_points[0])
center_point += shift[1] * numpy.subtract(square_points[3], square_points[0])
scale = numpy.linalg.norm(center_point - square_points[0]) * coverage
top_bottom_direction = calc_points_direction(square_points[0], square_points[2]) * scale
bottom_top_direction = calc_points_direction(square_points[3], square_points[1]) * scale
source_points = numpy.array([ center_point - top_bottom_direction, center_point + bottom_top_direction, center_point + top_bottom_direction ]).astype(numpy.float32)
affine_matrix = estimate_matrix_by_points(source_points, 'triangle_skew', crop_size)
crop_vision_frame = cv2.warpAffine(temp_vision_frame, affine_matrix, crop_size, flags = cv2.INTER_CUBIC)
return crop_vision_frame, affine_matrix
def warp_face_by_bounding_box(temp_vision_frame : VisionFrame, bounding_box : BoundingBox, crop_size : Size) -> Tuple[VisionFrame, Matrix]:
source_points = numpy.array([ [ bounding_box[0], bounding_box[1] ], [bounding_box[2], bounding_box[1] ], [ bounding_box[0], bounding_box[3] ] ]).astype(numpy.float32)
affine_matrix = estimate_matrix_by_points(source_points, 'triangle_orthogonal', crop_size)
target_points = numpy.array([ [ 0, 0 ], [ crop_size[0], 0 ], [ 0, crop_size[1] ] ]).astype(numpy.float32)
affine_matrix = cv2.getAffineTransform(source_points, target_points)
if bounding_box[2] - bounding_box[0] > crop_size[0] or bounding_box[3] - bounding_box[1] > crop_size[1]:
interpolation_method = cv2.INTER_AREA
else:
@ -153,12 +118,6 @@ def paste_back(temp_vision_frame : VisionFrame, crop_vision_frame : VisionFrame,
return paste_vision_frame
def calc_points_direction(start_point : Points, end_point : Points) -> Direction:
direction = end_point - start_point
direction /= numpy.linalg.norm(direction)
return direction
@lru_cache(maxsize = None)
def create_static_anchors(feature_stride : int, anchor_total : int, stride_height : int, stride_width : int) -> Anchors:
y, x = numpy.mgrid[:stride_height, :stride_width][::-1]

22
facefusion/processors/modules/deep_swapper.py
View File

@ -10,7 +10,7 @@ import facefusion.processors.core as processors
from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, wording
from facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url_by_provider
from facefusion.face_analyser import get_many_faces, get_one_face
from facefusion.face_helper import paste_back, warp_face_for_deepfacelive
from facefusion.face_helper import paste_back, warp_face_by_face_landmark_5
from facefusion.face_masker import create_occlusion_mask, create_static_box_mask
from facefusion.face_selector import find_similar_faces, sort_and_filter_faces
from facefusion.face_store import get_reference_faces
@ -44,9 +44,8 @@ def create_model_set() -> ModelSet:
'path': resolve_relative_path('../.assets/models/iperov/emma_watson_224.dfm')
}
},
'template': 'dfl_whole_face',
'size': (224, 224),
'shift': (0.0, 0.0),
'coverage': 2.2
},
'iperov/jackie_chan_224':
{
@ -66,9 +65,8 @@ def create_model_set() -> ModelSet:
'path': resolve_relative_path('../.assets/models/iperov/jackie_chan_224.dfm')
}
},
'template': 'dfl_whole_face',
'size': (224, 224),
'shift': (0.0, 0.0),
'coverage': 2.2
},
'iperov/keanu_reeves_320':
{
@ -88,9 +86,8 @@ def create_model_set() -> ModelSet:
'path': resolve_relative_path('../.assets/models/iperov/keanu_reeves_320.dfm')
}
},
'template': 'dfl_whole_face',
'size': (320, 320),
'shift': (0.0, 0.0),
'coverage': 2.2
},
'iperov/sylvester_stallone_224':
{
@ -110,9 +107,8 @@ def create_model_set() -> ModelSet:
'path': resolve_relative_path('../.assets/models/iperov/sylvester_stallone_224.dfm')
}
},
'template': 'dfl_whole_face',
'size': (224, 224),
'shift': (0.0, 0.0),
'coverage': 2.2
},
'iperov/taylor_swift_224':
{
@ -132,9 +128,8 @@ def create_model_set() -> ModelSet:
'path': resolve_relative_path('../.assets/models/iperov/taylor_swift_224.dfm')
}
},
'template': 'dfl_whole_face',
'size': (224, 224),
'shift': (0.0, 0.0),
'coverage': 2.2
}
}
@ -200,10 +195,9 @@ def post_process() -> None:
def swap_face(target_face : Face, temp_vision_frame : VisionFrame) -> VisionFrame:
model_template = get_model_options().get('template')
model_size = get_model_options().get('size')
model_shift = get_model_options().get('shift')
model_coverage = get_model_options().get('coverage')
crop_vision_frame, affine_matrix = warp_face_for_deepfacelive(temp_vision_frame, target_face.landmark_set.get('5/68'), model_size, model_shift, model_coverage)
crop_vision_frame, affine_matrix = warp_face_by_face_landmark_5(temp_vision_frame, target_face.landmark_set.get('5/68'), model_template, model_size)
crop_vision_frame_raw = crop_vision_frame.copy()
box_mask = create_static_box_mask(crop_vision_frame.shape[:2][::-1], state_manager.get_item('face_mask_blur'), state_manager.get_item('face_mask_padding'))
crop_masks =\

5
facefusion/typing.py
View File

@ -53,7 +53,6 @@ FaceStore = TypedDict('FaceStore',
VisionFrame = NDArray[Any]
Mask = NDArray[Any]
Points = NDArray[Any]
Direction = NDArray[Any]
Distance = NDArray[Any]
Matrix = NDArray[Any]
Anchors = NDArray[Any]
@ -86,10 +85,8 @@ ProcessStep = Callable[[str, int, Args], bool]
Content = Dict[str, Any]
WarpTemplate = Literal['arcface_112_v1', 'arcface_112_v2', 'arcface_128_v2', 'deepfacelive', 'ffhq_512', 'mtcnn_512', 'styleganex_384']
WarpTemplate = Literal['arcface_112_v1', 'arcface_112_v2', 'arcface_128_v2', 'dfl_head', 'dfl_whole_face', 'ffhq_512', 'mtcnn_512', 'styleganex_384']
WarpTemplateSet = Dict[WarpTemplate, NDArray[Any]]
PointsTemplate = Literal['square', 'triangle_orthogonal', 'triangle_skew']
PointsTemplateSet = Dict[PointsTemplate, NDArray[Any]]
ProcessMode = Literal['output', 'preview', 'stream']
ErrorCode = Literal[0, 1, 2, 3, 4]