mirror of
https://github.com/TencentARC/GFPGAN.git
synced 2025-05-17 07:40:16 -07:00
419 lines
16 KiB
Python
419 lines
16 KiB
Python
import math
|
|
import random
|
|
import torch
|
|
from torch import nn
|
|
from torch.nn import functional as F
|
|
|
|
from basicsr.archs.stylegan2_arch import (ConvLayer, EqualConv2d, EqualLinear, ResBlock, ScaledLeakyReLU,
|
|
StyleGAN2Generator)
|
|
from basicsr.ops.fused_act import FusedLeakyReLU
|
|
from basicsr.utils.registry import ARCH_REGISTRY
|
|
|
|
|
|
class StyleGAN2GeneratorSFT(StyleGAN2Generator):
|
|
"""StyleGAN2 Generator.
|
|
|
|
Args:
|
|
out_size (int): The spatial size of outputs.
|
|
num_style_feat (int): Channel number of style features. Default: 512.
|
|
num_mlp (int): Layer number of MLP style layers. Default: 8.
|
|
channel_multiplier (int): Channel multiplier for large networks of
|
|
StyleGAN2. Default: 2.
|
|
resample_kernel (list[int]): A list indicating the 1D resample kernel
|
|
magnitude. A cross production will be applied to extent 1D resample
|
|
kenrel to 2D resample kernel. Default: [1, 3, 3, 1].
|
|
lr_mlp (float): Learning rate multiplier for mlp layers. Default: 0.01.
|
|
"""
|
|
|
|
def __init__(self,
|
|
out_size,
|
|
num_style_feat=512,
|
|
num_mlp=8,
|
|
channel_multiplier=2,
|
|
resample_kernel=(1, 3, 3, 1),
|
|
lr_mlp=0.01,
|
|
narrow=1,
|
|
sft_half=False):
|
|
super(StyleGAN2GeneratorSFT, self).__init__(
|
|
out_size,
|
|
num_style_feat=num_style_feat,
|
|
num_mlp=num_mlp,
|
|
channel_multiplier=channel_multiplier,
|
|
resample_kernel=resample_kernel,
|
|
lr_mlp=lr_mlp,
|
|
narrow=narrow)
|
|
self.sft_half = sft_half
|
|
|
|
def forward(self,
|
|
styles,
|
|
conditions,
|
|
input_is_latent=False,
|
|
noise=None,
|
|
randomize_noise=True,
|
|
truncation=1,
|
|
truncation_latent=None,
|
|
inject_index=None,
|
|
return_latents=False):
|
|
"""Forward function for StyleGAN2Generator.
|
|
|
|
Args:
|
|
styles (list[Tensor]): Sample codes of styles.
|
|
input_is_latent (bool): Whether input is latent style.
|
|
Default: False.
|
|
noise (Tensor | None): Input noise or None. Default: None.
|
|
randomize_noise (bool): Randomize noise, used when 'noise' is
|
|
False. Default: True.
|
|
truncation (float): TODO. Default: 1.
|
|
truncation_latent (Tensor | None): TODO. Default: None.
|
|
inject_index (int | None): The injection index for mixing noise.
|
|
Default: None.
|
|
return_latents (bool): Whether to return style latents.
|
|
Default: False.
|
|
"""
|
|
# style codes -> latents with Style MLP layer
|
|
if not input_is_latent:
|
|
styles = [self.style_mlp(s) for s in styles]
|
|
# noises
|
|
if noise is None:
|
|
if randomize_noise:
|
|
noise = [None] * self.num_layers # for each style conv layer
|
|
else: # use the stored noise
|
|
noise = [getattr(self.noises, f'noise{i}') for i in range(self.num_layers)]
|
|
# style truncation
|
|
if truncation < 1:
|
|
style_truncation = []
|
|
for style in styles:
|
|
style_truncation.append(truncation_latent + truncation * (style - truncation_latent))
|
|
styles = style_truncation
|
|
# get style latent with injection
|
|
if len(styles) == 1:
|
|
inject_index = self.num_latent
|
|
|
|
if styles[0].ndim < 3:
|
|
# repeat latent code for all the layers
|
|
latent = styles[0].unsqueeze(1).repeat(1, inject_index, 1)
|
|
else: # used for encoder with different latent code for each layer
|
|
latent = styles[0]
|
|
elif len(styles) == 2: # mixing noises
|
|
if inject_index is None:
|
|
inject_index = random.randint(1, self.num_latent - 1)
|
|
latent1 = styles[0].unsqueeze(1).repeat(1, inject_index, 1)
|
|
latent2 = styles[1].unsqueeze(1).repeat(1, self.num_latent - inject_index, 1)
|
|
latent = torch.cat([latent1, latent2], 1)
|
|
|
|
# main generation
|
|
out = self.constant_input(latent.shape[0])
|
|
out = self.style_conv1(out, latent[:, 0], noise=noise[0])
|
|
skip = self.to_rgb1(out, latent[:, 1])
|
|
|
|
i = 1
|
|
for conv1, conv2, noise1, noise2, to_rgb in zip(self.style_convs[::2], self.style_convs[1::2], noise[1::2],
|
|
noise[2::2], self.to_rgbs):
|
|
out = conv1(out, latent[:, i], noise=noise1)
|
|
|
|
# the conditions may have fewer levels
|
|
if i < len(conditions):
|
|
# SFT part to combine the conditions
|
|
if self.sft_half:
|
|
out_same, out_sft = torch.split(out, int(out.size(1) // 2), dim=1)
|
|
out_sft = out_sft * conditions[i - 1] + conditions[i]
|
|
out = torch.cat([out_same, out_sft], dim=1)
|
|
else:
|
|
out = out * conditions[i - 1] + conditions[i]
|
|
|
|
out = conv2(out, latent[:, i + 1], noise=noise2)
|
|
skip = to_rgb(out, latent[:, i + 2], skip)
|
|
i += 2
|
|
|
|
image = skip
|
|
|
|
if return_latents:
|
|
return image, latent
|
|
else:
|
|
return image, None
|
|
|
|
|
|
class ConvUpLayer(nn.Module):
|
|
"""Conv Up Layer. Bilinear upsample + Conv.
|
|
|
|
Args:
|
|
in_channels (int): Channel number of the input.
|
|
out_channels (int): Channel number of the output.
|
|
kernel_size (int): Size of the convolving kernel.
|
|
stride (int): Stride of the convolution. Default: 1
|
|
padding (int): Zero-padding added to both sides of the input.
|
|
Default: 0.
|
|
bias (bool): If ``True``, adds a learnable bias to the output.
|
|
Default: ``True``.
|
|
bias_init_val (float): Bias initialized value. Default: 0.
|
|
activate (bool): Whether use activateion. Default: True.
|
|
"""
|
|
|
|
def __init__(self,
|
|
in_channels,
|
|
out_channels,
|
|
kernel_size,
|
|
stride=1,
|
|
padding=0,
|
|
bias=True,
|
|
bias_init_val=0,
|
|
activate=True):
|
|
super(ConvUpLayer, self).__init__()
|
|
self.in_channels = in_channels
|
|
self.out_channels = out_channels
|
|
self.kernel_size = kernel_size
|
|
self.stride = stride
|
|
self.padding = padding
|
|
self.scale = 1 / math.sqrt(in_channels * kernel_size**2)
|
|
|
|
self.weight = nn.Parameter(torch.randn(out_channels, in_channels, kernel_size, kernel_size))
|
|
|
|
if bias and not activate:
|
|
self.bias = nn.Parameter(torch.zeros(out_channels).fill_(bias_init_val))
|
|
else:
|
|
self.register_parameter('bias', None)
|
|
|
|
# activation
|
|
if activate:
|
|
if bias:
|
|
self.activation = FusedLeakyReLU(out_channels)
|
|
else:
|
|
self.activation = ScaledLeakyReLU(0.2)
|
|
else:
|
|
self.activation = None
|
|
|
|
def forward(self, x):
|
|
# bilinear upsample
|
|
out = F.interpolate(x, scale_factor=2, mode='bilinear', align_corners=False)
|
|
# conv
|
|
out = F.conv2d(
|
|
out,
|
|
self.weight * self.scale,
|
|
bias=self.bias,
|
|
stride=self.stride,
|
|
padding=self.padding,
|
|
)
|
|
# activation
|
|
if self.activation is not None:
|
|
out = self.activation(out)
|
|
return out
|
|
|
|
|
|
class ResUpBlock(nn.Module):
|
|
"""Residual block with upsampling.
|
|
|
|
Args:
|
|
in_channels (int): Channel number of the input.
|
|
out_channels (int): Channel number of the output.
|
|
"""
|
|
|
|
def __init__(self, in_channels, out_channels):
|
|
super(ResUpBlock, self).__init__()
|
|
|
|
self.conv1 = ConvLayer(in_channels, in_channels, 3, bias=True, activate=True)
|
|
self.conv2 = ConvUpLayer(in_channels, out_channels, 3, stride=1, padding=1, bias=True, activate=True)
|
|
self.skip = ConvUpLayer(in_channels, out_channels, 1, bias=False, activate=False)
|
|
|
|
def forward(self, x):
|
|
out = self.conv1(x)
|
|
out = self.conv2(out)
|
|
skip = self.skip(x)
|
|
out = (out + skip) / math.sqrt(2)
|
|
return out
|
|
|
|
|
|
@ARCH_REGISTRY.register()
|
|
class GFPGANv1(nn.Module):
|
|
"""Unet + StyleGAN2 decoder with SFT."""
|
|
|
|
def __init__(
|
|
self,
|
|
out_size,
|
|
num_style_feat=512,
|
|
channel_multiplier=1,
|
|
resample_kernel=(1, 3, 3, 1),
|
|
decoder_load_path=None,
|
|
fix_decoder=True,
|
|
# for stylegan decoder
|
|
num_mlp=8,
|
|
lr_mlp=0.01,
|
|
input_is_latent=False,
|
|
different_w=False,
|
|
narrow=1,
|
|
sft_half=False):
|
|
|
|
super(GFPGANv1, self).__init__()
|
|
self.input_is_latent = input_is_latent
|
|
self.different_w = different_w
|
|
self.num_style_feat = num_style_feat
|
|
|
|
unet_narrow = narrow * 0.5
|
|
channels = {
|
|
'4': int(512 * unet_narrow),
|
|
'8': int(512 * unet_narrow),
|
|
'16': int(512 * unet_narrow),
|
|
'32': int(512 * unet_narrow),
|
|
'64': int(256 * channel_multiplier * unet_narrow),
|
|
'128': int(128 * channel_multiplier * unet_narrow),
|
|
'256': int(64 * channel_multiplier * unet_narrow),
|
|
'512': int(32 * channel_multiplier * unet_narrow),
|
|
'1024': int(16 * channel_multiplier * unet_narrow)
|
|
}
|
|
|
|
self.log_size = int(math.log(out_size, 2))
|
|
first_out_size = 2**(int(math.log(out_size, 2)))
|
|
|
|
self.conv_body_first = ConvLayer(3, channels[f'{first_out_size}'], 1, bias=True, activate=True)
|
|
|
|
# downsample
|
|
in_channels = channels[f'{first_out_size}']
|
|
self.conv_body_down = nn.ModuleList()
|
|
for i in range(self.log_size, 2, -1):
|
|
out_channels = channels[f'{2**(i - 1)}']
|
|
self.conv_body_down.append(ResBlock(in_channels, out_channels, resample_kernel))
|
|
in_channels = out_channels
|
|
|
|
self.final_conv = ConvLayer(in_channels, channels['4'], 3, bias=True, activate=True)
|
|
|
|
# upsample
|
|
in_channels = channels['4']
|
|
self.conv_body_up = nn.ModuleList()
|
|
for i in range(3, self.log_size + 1):
|
|
out_channels = channels[f'{2**i}']
|
|
self.conv_body_up.append(ResUpBlock(in_channels, out_channels))
|
|
in_channels = out_channels
|
|
|
|
# to RGB
|
|
self.toRGB = nn.ModuleList()
|
|
for i in range(3, self.log_size + 1):
|
|
self.toRGB.append(EqualConv2d(channels[f'{2**i}'], 3, 1, stride=1, padding=0, bias=True, bias_init_val=0))
|
|
|
|
if different_w:
|
|
linear_out_channel = (int(math.log(out_size, 2)) * 2 - 2) * num_style_feat
|
|
else:
|
|
linear_out_channel = num_style_feat
|
|
|
|
self.final_linear = EqualLinear(
|
|
channels['4'] * 4 * 4, linear_out_channel, bias=True, bias_init_val=0, lr_mul=1, activation=None)
|
|
|
|
self.stylegan_decoder = StyleGAN2GeneratorSFT(
|
|
out_size=out_size,
|
|
num_style_feat=num_style_feat,
|
|
num_mlp=num_mlp,
|
|
channel_multiplier=channel_multiplier,
|
|
resample_kernel=resample_kernel,
|
|
lr_mlp=lr_mlp,
|
|
narrow=narrow,
|
|
sft_half=sft_half)
|
|
|
|
if decoder_load_path:
|
|
self.stylegan_decoder.load_state_dict(
|
|
torch.load(decoder_load_path, map_location=lambda storage, loc: storage)['params_ema'])
|
|
if fix_decoder:
|
|
for _, param in self.stylegan_decoder.named_parameters():
|
|
param.requires_grad = False
|
|
|
|
# for SFT
|
|
self.condition_scale = nn.ModuleList()
|
|
self.condition_shift = nn.ModuleList()
|
|
for i in range(3, self.log_size + 1):
|
|
out_channels = channels[f'{2**i}']
|
|
if sft_half:
|
|
sft_out_channels = out_channels
|
|
else:
|
|
sft_out_channels = out_channels * 2
|
|
self.condition_scale.append(
|
|
nn.Sequential(
|
|
EqualConv2d(out_channels, out_channels, 3, stride=1, padding=1, bias=True, bias_init_val=0),
|
|
ScaledLeakyReLU(0.2),
|
|
EqualConv2d(out_channels, sft_out_channels, 3, stride=1, padding=1, bias=True, bias_init_val=1)))
|
|
self.condition_shift.append(
|
|
nn.Sequential(
|
|
EqualConv2d(out_channels, out_channels, 3, stride=1, padding=1, bias=True, bias_init_val=0),
|
|
ScaledLeakyReLU(0.2),
|
|
EqualConv2d(out_channels, sft_out_channels, 3, stride=1, padding=1, bias=True, bias_init_val=0)))
|
|
|
|
def forward(self,
|
|
x,
|
|
return_latents=False,
|
|
save_feat_path=None,
|
|
load_feat_path=None,
|
|
return_rgb=True,
|
|
randomize_noise=True):
|
|
conditions = []
|
|
unet_skips = []
|
|
out_rgbs = []
|
|
|
|
# encoder
|
|
feat = self.conv_body_first(x)
|
|
for i in range(self.log_size - 2):
|
|
feat = self.conv_body_down[i](feat)
|
|
unet_skips.insert(0, feat)
|
|
|
|
feat = self.final_conv(feat)
|
|
|
|
# style code
|
|
style_code = self.final_linear(feat.view(feat.size(0), -1))
|
|
if self.different_w:
|
|
style_code = style_code.view(style_code.size(0), -1, self.num_style_feat)
|
|
|
|
# decode
|
|
for i in range(self.log_size - 2):
|
|
# add unet skip
|
|
feat = feat + unet_skips[i]
|
|
# ResUpLayer
|
|
feat = self.conv_body_up[i](feat)
|
|
# generate scale and shift for SFT layer
|
|
scale = self.condition_scale[i](feat)
|
|
conditions.append(scale.clone())
|
|
shift = self.condition_shift[i](feat)
|
|
conditions.append(shift.clone())
|
|
# generate rgb images
|
|
if return_rgb:
|
|
out_rgbs.append(self.toRGB[i](feat))
|
|
|
|
if save_feat_path is not None:
|
|
torch.save(conditions, save_feat_path)
|
|
if load_feat_path is not None:
|
|
conditions = torch.load(load_feat_path)
|
|
conditions = [v.cuda() for v in conditions]
|
|
|
|
# decoder
|
|
image, _ = self.stylegan_decoder([style_code],
|
|
conditions,
|
|
return_latents=return_latents,
|
|
input_is_latent=self.input_is_latent,
|
|
randomize_noise=randomize_noise)
|
|
|
|
return image, out_rgbs
|
|
|
|
|
|
@ARCH_REGISTRY.register()
|
|
class FacialComponentDiscriminator(nn.Module):
|
|
|
|
def __init__(self):
|
|
super(FacialComponentDiscriminator, self).__init__()
|
|
|
|
self.conv1 = ConvLayer(3, 64, 3, downsample=False, resample_kernel=(1, 3, 3, 1), bias=True, activate=True)
|
|
self.conv2 = ConvLayer(64, 128, 3, downsample=True, resample_kernel=(1, 3, 3, 1), bias=True, activate=True)
|
|
self.conv3 = ConvLayer(128, 128, 3, downsample=False, resample_kernel=(1, 3, 3, 1), bias=True, activate=True)
|
|
self.conv4 = ConvLayer(128, 256, 3, downsample=True, resample_kernel=(1, 3, 3, 1), bias=True, activate=True)
|
|
self.conv5 = ConvLayer(256, 256, 3, downsample=False, resample_kernel=(1, 3, 3, 1), bias=True, activate=True)
|
|
self.final_conv = ConvLayer(256, 1, 3, bias=True, activate=False)
|
|
|
|
def forward(self, x, return_feats=False):
|
|
feat = self.conv1(x)
|
|
feat = self.conv3(self.conv2(feat))
|
|
rlt_feats = []
|
|
if return_feats:
|
|
rlt_feats.append(feat.clone())
|
|
feat = self.conv5(self.conv4(feat))
|
|
if return_feats:
|
|
rlt_feats.append(feat.clone())
|
|
out = self.final_conv(feat)
|
|
|
|
if return_feats:
|
|
return out, rlt_feats
|
|
else:
|
|
return out, None
|