# Finetuning Donut Transformer on DocParsing

**URL:** <https://discuss.huggingface.co/t/finetuning-donut-transformer-on-docparsing/59547>\
**Category:** Beginners\
**Created:** [October 23, 2023, 11:08am UTC](https://discuss.huggingface.co/t/finetuning-donut-transformer-on-docparsing/59547 "2023-10-23T11:08:52Z")\
**Posts on this page:** 1\
**Page:** 1

<div class="post-metadata">

**Author:** ![anon92062415](https://avatars.discourse-cdn.com/v4/letter/a/e274bd/32.png) [@anon92062415](https://discuss.huggingface.co/u/anon92062415)\
**Post date:** [October 23, 2023, 11:08am UTC](https://discuss.huggingface.co/t/finetuning-donut-transformer-on-docparsing/59547/1 "2023-10-23T11:08:52Z")

</div>

Hi everyone,  
I am trying to finetune the donut transformer on docparsing for a batch commercial invoice documents that are used in the banking sector. However; I am constantly receiving the below error, even after a significant amount of research. If you can direct me about how to solve the error, I will be more than thankful.

I tried to change the arguments of the trainer, data collator, and training args objects. I also tried to change the DataCollatorWithPadding() object to DefaultDataCollator() after receiving an error about padding. Moreover, for constructing the tokenizer object, I also tried to use AutoTokenizer.fromPretrained() instead of processor.tokenizer(). Lastly, I checked the HuggingFace documentation for Donut Transformer. However, nothing changed. I continue to receive the below error.

Best regards,  
Barış

**The code that I have written:**

import re  
import sys  
from PIL import Image  
import torch  
import os  
import ssl  
import glob  
from datasets import load\_dataset  
from transformers import DonutProcessor, VisionEncoderDecoderModel, AutoTokenizer, AutoFeatureExtractor, AutoFeatureExtractor  
from transformers import TrainingArguments, Trainer, DefaultDataCollator

# Disabling SSL verification and configuring environment variables

ssl.\_create\_default\_https\_context = ssl.\_create\_unverified\_context  
os.environ[“REQUESTS\_CA\_BUNDLE”] = “/etc/ssl/certs/ca-bundle.trust.crt”  
os.environ[“TF\_ENABLE\_ONEDNN\_OPTS”] = “0”

def fine\_tune\_donut\_model():  
# Load the pre-trained Donut processor and model as a base for fine-tuning  
processor = DonutProcessor.from\_pretrained(“naver-clova-ix/donut-base”)  
model = VisionEncoderDecoderModel.from\_pretrained(“naver-clova-ix/donut-base”)

```
# Inside the `fine_tune_donut_model` function, after loading the dataset
dataset = load_dataset("/home/t098317/donut_directory/donut/commercial_invoice")

# Load a pretrained tokenizer for text data
# tokenizer = AutoTokenizer.from_pretrained("naver-clova-ix/donut-base")

task_prompt = "<s_cord-v2>"
tokenizer = processor.tokenizer(task_prompt, add_special_tokens=False, return_tensors="pt")
print('the tokenizer is: '+str(tokenizer)+'')

# Check if a CUDA-enabled GPU is available, otherwise use CPU
device = "cuda" if torch.cuda.is_available() else "cpu"
model.to(device)

# Define your fine-tuning hyperparameters and training arguments
training_args = TrainingArguments(
    output_dir="/home/t098317/donut_directory/donut/finetuned_donut",
    per_device_train_batch_size=8,
    num_train_epochs=3,
    evaluation_strategy="steps",
    save_steps=10_000,
    remove_unused_columns=False,
    learning_rate=0.0001
)

print('The training args are: '+str(training_args)+'')
print()
print()

# Create a data collator for your training data
data_collator = DefaultDataCollator(
    return_tensors="pt"
)

print('The data collator is: '+str(data_collator)+'')
print()
print()

trainer = Trainer(
    tokenizer=tokenizer,
    model=model,
    args=training_args,
    data_collator=data_collator,  
    train_dataset=dataset["train"]
)

print('The trainer is: '+str(trainer)+'')
print()
print()

# Start the fine-tuning process
try:
# Start the fine-tuning process
    trainer.train()
except Exception as e:
    print(f"Error during training: {str(e)}")

print('abc')

# Save the fine-tuned model in the specified directory
model.save_pretrained("/home/t098317/donut_directory/donut/finetuned_donut")

```

# Function to extract information from all images in a directory

def extract\_info\_from\_images\_in\_directory(directory\_path, task\_name=“cord-v2”):  
# Load the fine-tuned Donut processor and model  
processor = DonutProcessor.from\_pretrained(“/home/t098317/donut\_directory/donut/finetuned\_donut”)  
model = VisionEncoderDecoderModel.from\_pretrained(“/home/t098317/donut\_directory/donut/finetuned\_donut”)

```
# Check if a CUDA-enabled GPU is available, otherwise use CPU
device = "cuda" if torch.cuda.is_available() else "cpu"
model.to(device)

# Get a list of image files in the specified directory
image_extensions = ['*.jpg', '*.png']

# Use glob to find all image files with the specified extensions
image_files = []
for ext in image_extensions:
    image_files.extend(glob.glob(os.path.join(directory_path, ext)))

for image_path in image_files:
    print(image_path)
    # Load the image using PIL (Python Imaging Library)
    input_img = Image.open(image_path)

    # Prepare the image input for Donut
    pixel_values = processor(input_img, return_tensors="pt")

    # Prepare decoder inputs
    task_prompt = f"<s_{task_name}>"
    decoder_input_ids = processor.tokenizer(task_prompt, add_special_tokens=False, return_tensors="pt").input_ids

    # Generate text from the image and decoder inputs
    outputs = model.generate(
        pixel_values.to(device),
        input_ids=decoder_input_ids.to(device), # Use input_ids instead of decoder_input_ids
        pad_token_id=processor.tokenizer.pad_token_id,
        eos_token_id=processor.tokenizer.eos_token_id,
        use_cache=True,
        max_new_tokens=700,
        bad_words_ids=[[processor.tokenizer.unk_token_id]],
        return_dict_in_generate=True,
        decoder_start_token_id=model.config.decoder_start_token_id, # Add this line if necessary
    )
    
    print('output: '+str(outputs)+'')

    # Decode the generated sequence
    sequence = processor.batch_decode(outputs.sequences)[0]
    sequence = sequence.replace(processor.tokenizer.eos_token, "").replace(processor.tokenizer.pad_token, "")
    sequence = re.sub(r"<.*?>", "", sequence, count=1).strip() # Remove the first task start token

    # Print the extracted information
    print(f"Information from {image_path}:")
    print(processor.token2json(sequence))
    print()

```

if **name** == “ **main** ”:  
# Execute the fine-tuning function  
fine\_tune\_donut\_model()

```
# Specify the directory containing the images you want to extract information from
image_directory_path = "/home/t098317/donut_directory/donut/commercial_invoice"

# Execute the information extraction function for all images in the directory with task name "cord-v2"
extract_info_from_images_in_directory(image_directory_path, task_name="cord-v2")

```

**The error I am receiving:**

Could not find image processor class in the image processor config or the model config. Loading based on pattern matching with the model’s feature extractor configuration.  
the tokenizer is: {‘input\_ids’: tensor([[41040, 46192, 41403, 36697, 52165, 37093, 35934, 34791]]), ‘attention\_mask’: tensor([[1, 1, 1, 1, 1, 1, 1, 1]])}  
The training args are: TrainingArguments(  
\_n\_gpu=3,  
adafactor=False,  
adam\_beta1=0.9,  
adam\_beta2=0.999,  
adam\_epsilon=1e-08,  
auto\_find\_batch\_size=False,  
bf16=False,  
bf16\_full\_eval=False,  
data\_seed=None,  
dataloader\_drop\_last=False,  
dataloader\_num\_workers=0,  
dataloader\_pin\_memory=True,  
ddp\_backend=None,  
ddp\_broadcast\_buffers=None,  
ddp\_bucket\_cap\_mb=None,  
ddp\_find\_unused\_parameters=None,  
ddp\_timeout=1800,  
debug=,  
deepspeed=None,  
disable\_tqdm=False,  
dispatch\_batches=None,  
do\_eval=True,  
do\_predict=False,  
do\_train=False,  
eval\_accumulation\_steps=None,  
eval\_delay=0,  
eval\_steps=500,  
evaluation\_strategy=steps,  
fp16=False,  
fp16\_backend=auto,  
fp16\_full\_eval=False,  
fp16\_opt\_level=O1,  
fsdp=,  
fsdp\_config={‘min\_num\_params’: 0, ‘xla’: False, ‘xla\_fsdp\_grad\_ckpt’: False},  
fsdp\_min\_num\_params=0,  
fsdp\_transformer\_layer\_cls\_to\_wrap=None,  
full\_determinism=False,  
gradient\_accumulation\_steps=1,  
gradient\_checkpointing=False,  
greater\_is\_better=None,  
group\_by\_length=False,  
half\_precision\_backend=auto,  
hub\_always\_push=False,  
hub\_model\_id=None,  
hub\_private\_repo=False,  
hub\_strategy=every\_save,  
hub\_token=\<HUB\_TOKEN\>,  
ignore\_data\_skip=False,  
include\_inputs\_for\_metrics=False,  
include\_tokens\_per\_second=False,  
jit\_mode\_eval=False,  
label\_names=None,  
label\_smoothing\_factor=0.0,  
learning\_rate=0.0001,  
length\_column\_name=length,  
load\_best\_model\_at\_end=False,  
local\_rank=0,  
log\_level=passive,  
log\_level\_replica=warning,  
log\_on\_each\_node=True,  
logging\_dir=/home/t098317/donut\_directory/donut/finetuned\_donut/runs/Oct23\_14-07-30\_ovrargegpudev1,  
logging\_first\_step=False,  
logging\_nan\_inf\_filter=True,  
logging\_steps=500,  
logging\_strategy=steps,  
lr\_scheduler\_type=linear,  
max\_grad\_norm=1.0,  
max\_steps=-1,  
metric\_for\_best\_model=None,  
mp\_parameters=,  
no\_cuda=False,  
num\_train\_epochs=3,  
optim=adamw\_torch,  
optim\_args=None,  
output\_dir=/home/t098317/donut\_directory/donut/finetuned\_donut,  
overwrite\_output\_dir=False,  
past\_index=-1,  
per\_device\_eval\_batch\_size=8,  
per\_device\_train\_batch\_size=8,  
prediction\_loss\_only=False,  
push\_to\_hub=False,  
push\_to\_hub\_model\_id=None,  
push\_to\_hub\_organization=None,  
push\_to\_hub\_token=\<PUSH\_TO\_HUB\_TOKEN\>,  
ray\_scope=last,  
remove\_unused\_columns=False,  
report\_to=[‘tensorboard’],  
resume\_from\_checkpoint=None,  
run\_name=/home/t098317/donut\_directory/donut/finetuned\_donut,  
save\_on\_each\_node=False,  
save\_safetensors=False,  
save\_steps=10000,  
save\_strategy=steps,  
save\_total\_limit=None,  
seed=42,  
sharded\_ddp=,  
skip\_memory\_metrics=True,  
tf32=None,  
torch\_compile=False,  
torch\_compile\_backend=None,  
torch\_compile\_mode=None,  
torchdynamo=None,  
tpu\_metrics\_debug=False,  
tpu\_num\_cores=None,  
use\_cpu=False,  
use\_ipex=False,  
use\_legacy\_prediction\_loop=False,  
use\_mps\_device=False,  
warmup\_ratio=0.0,  
warmup\_steps=0,  
weight\_decay=0.0,  
)

The data collator is: DefaultDataCollator(return\_tensors=‘pt’)

The trainer is: \<transformers.trainer.Trainer object at 0x7f4d26040ac0\>

0%| | 0/3 [00:00\<?, ?it/s]Error during training: Could not infer dtype of JpegImageFile  
abc  
0%| | 0/3 [00:03\<?, ?it/s]  
Traceback (most recent call last):  
File “/home/t098317/donut\_directory/donut/finetune\_donut\_dp.py”, line 149, in   
extract\_info\_from\_images\_in\_directory(image\_directory\_path, task\_name=“cord-v2”)  
File “/home/t098317/donut\_directory/donut/finetune\_donut\_dp.py”, line 88, in extract\_info\_from\_images\_in\_directory  
processor = DonutProcessor.from\_pretrained(“/home/t098317/donut\_directory/donut/finetuned\_donut”)  
File “/storage/miniconda/envs/yagmur\_donut/lib/python3.9/site-packages/transformers/processing\_utils.py”, line 226, in from\_pretrained  
args = cls.\_get\_arguments\_from\_pretrained(pretrained\_model\_name\_or\_path, \*\*kwargs)  
File “/storage/miniconda/envs/yagmur\_donut/lib/python3.9/site-packages/transformers/processing\_utils.py”, line 270, in \_get\_arguments\_from\_pretrained  
args.append(attribute\_class.from\_pretrained(pretrained\_model\_name\_or\_path, \*\*kwargs))  
File “/storage/miniconda/envs/yagmur\_donut/lib/python3.9/site-packages/transformers/models/auto/image\_processing\_auto.py”, line 346, in from\_pretrained  
config\_dict, \_ = ImageProcessingMixin.get\_image\_processor\_dict(pretrained\_model\_name\_or\_path, \*\*kwargs)  
File “/storage/miniconda/envs/yagmur\_donut/lib/python3.9/site-packages/transformers/image\_processing\_utils.py”, line 329, in get\_image\_processor\_dict  
resolved\_image\_processor\_file = cached\_file(  
File “/storage/miniconda/envs/yagmur\_donut/lib/python3.9/site-packages/transformers/utils/hub.py”, line 400, in cached\_file  
raise EnvironmentError(  
OSError: /home/t098317/donut\_directory/donut/finetuned\_donut does not appear to have a file named preprocessor\_config.json. Checkout ‘[https://huggingface.co//home/t098317/donut\_directory/donut/finetuned\_donut/main](https://huggingface.co//home/t098317/donut_directory/donut/finetuned_donut/main)’ for available files.