@InProceedings{HectorLariaMantecon2024,
author="Hector Laria Mantecon
and Kai Wang
and Joost Van de Weijer
and Bogdan Raducanu",
title="NeRF-Diffusion for 3D-Consistent Face Generation and Editing",
booktitle="19th International Joint Conference on Computer Vision, Imaging and Computer Graphics Theory and Applications",
year="2024",
abstract="Generating high-fidelity 3D-aware images without 3D supervision is a valuable capability in various applications. Current methods based on NeRF features, SDF information, or triplane features have limited variation after training. To address this, we propose a novel approach that combines pretrained models for shape and content generation. Our method leverages a pretrained Neural Radiance Field as a shape prior and a diffusion model for content generation. By conditioning the diffusion model with 3D features, we enhance its ability to generate novel views with 3D awareness. We introduce a consistency token shared between the NeRF module and the diffusion model to maintain 3D consistency during sampling. Moreover, our framework allows for text editing of 3D-aware image generation, enabling users to modify the style over 3D views while preserving semantic content. Our contributions include incorporating 3D awareness into a text-to-image model, addressing identity consistency in 3D view synthesis, and enabling text editing of 3D-aware image generation. We provide detailed explanations, including the shape prior based on the NeRF model and the content generation process using the diffusion model. We also discuss challenges such as shape consistency and sampling saturation. Experimental results demonstrate the effectiveness and visual quality of our approach.",
optnote="LAMP",
optnote="exported from refbase (http://refbase.cvc.uab.es/show.php?record=4003), last updated on Tue, 12 Mar 2024 12:26:15 +0100",
opturl="https://www.insticc.org/node/TechnicalProgram/visigrapp/2024/presentationDetails/123815"
}