@Misc{SouhailBakkali2023,
author="Souhail Bakkali
and Sanket Biswas
and Zuheng Ming
and Mickael Coustaty
and Mar{\c{c}}al Rusi{\~n}ol
and Oriol Ramos Terrades
and Josep Llados",
title="TransferDoc: A Self-Supervised Transferable Document Representation Learning Model Unifying Vision and Language",
year="2023",
abstract="The field of visual document understanding has witnessed a rapid growth in emerging challenges and powerful multi-modal strategies. However, they rely on an extensive amount of document data to learn their pretext objectives in a {\textquoteleft}{\textquoteleft}pre-train-then-fine-tune{\textquoteright}{\textquoteright} paradigm and thus, suffer a significant performance drop in real-world online industrial settings. One major reason is the over-reliance on OCR engines to extract local positional information within a document page. Therefore, this hinders the model{\textquoteright}s generalizability, flexibility and robustness due to the lack of capturing global information within a document image. We introduce TransferDoc, a cross-modal transformer-based architecture pre-trained in a self-supervised fashion using three novel pretext objectives. TransferDoc learns richer semantic concepts by unifying language and visual representations, which enables the production of more transferable models. Besides, two novel downstream tasks have been introduced for a {\textquoteleft}{\textquoteleft}closer-to-real{\textquoteright}{\textquoteright} industrial evaluation scenario where TransferDoc outperforms other state-of-the-art approaches.",
optnote="DAG",
optnote="exported from refbase (http://refbase.cvc.uab.es/show.php?record=3995), last updated on Wed, 31 Jan 2024 10:52:33 +0100",
opturl="https://arxiv.org/abs/2309.05756",
file=":http://refbase.cvc.uab.es/files/BBM2023.pdf:PDF"
}