Duplicate from philschmid/donut-base-finetuned-cord-v2

Co-authored-by: Philipp Schmid <[email protected]>

.gitattributes

@@ -0,0 +1,32 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,80 @@
+---
+license: mit
+tags:
+- donut
+- image-to-text
+- vision
+- endpoints-template
+---
+
+# Fork of [naver-clova-ix/donut-base-finetuned-cord-v2](https://huggingface.co/naver-clova-ix/donut-base-finetuned-cord-v2)
+
+> This is fork of [naver-clova-ix/donut-base-finetuned-cord-v2](https://huggingface.co/naver-clova-ix/donut-base-finetuned-cord-v2) implementing a custom `handler.py` as an example for how to use `donut` models with [inference-endpoints](https://hf.co/inference-endpoints)
+
+--- 
+
+# Donut (base-sized model, fine-tuned on CORD) 
+
+Donut model fine-tuned on CORD. It was introduced in the paper [OCR-free Document Understanding Transformer](https://arxiv.org/abs/2111.15664) by Geewok et al. and first released in [this repository](https://github.com/clovaai/donut).
+
+Donut consists of a vision encoder (Swin Transformer) and a text decoder (BART). Given an image, the encoder first encodes the image into a tensor of embeddings (of shape batch_size, seq_len, hidden_size), after which the decoder autoregressively generates text, conditioned on the encoding of the encoder. 
+
+# Use with Inference Endpoints 
+
+Hugging Face Inference endpoints can directly work with binary data, this means that we can directly send our image from our document to the endpoint. We are going to use requests to send our requests. (make your you have it installed `pip install requests`)
+
+![result](res.png)
+
+## Send requests with Pyton
+
+load sample image
+
+```bash
+wget https://huggingface.co/philschmid/donut-base-finetuned-cord-v2/resolve/main/sample.png
+```
+
+send request to endpoint
+
+```python
+import json
+import requests as r
+import mimetypes
+
+ENDPOINT_URL="" # url of your endpoint
+HF_TOKEN="" # organization token where you deployed your endpoint
+
+def predict(path_to_image:str=None):
+    with open(path_to_image, "rb") as i:
+      b = i.read()
+    headers= {
+        "Authorization": f"Bearer {HF_TOKEN}",
+        "Content-Type": mimetypes.guess_type(path_to_image)[0]
+    }
+    response = r.post(ENDPOINT_URL, headers=headers, data=b)
+    return response.json()
+
+prediction = predict(path_to_image="sample.png")
+
+print(prediction)
+# {'menu': [{'nm': '0571-1854 BLUS WANITA',
+#   'unitprice': '@120.000',
+#   'cnt': '1',
+#   'price': '120,000'},
+#  {'nm': '1002-0060 SHOPPING BAG', 'cnt': '1', 'price': '0'}],
+# 'total': {'total_price': '120,000',
+#  'changeprice': '0',
+#  'creditcardprice': '120,000',
+#  'menuqty_cnt': '1'}}
+```
+
+
+
+**curl example**
+
+```bash
+curl https://ak7gduay2ypyr9vp.us-east-1.aws.endpoints.huggingface.cloud \
+-X POST \
+--data-binary 'sample.png' \
+-H "Authorization: Bearer XXX" \
+-H "Content-Type: null"
+```

added_tokens.json

@@ -0,0 +1,60 @@
+{
+  "</s_cashprice>": 57549,
+  "</s_changeprice>": 57551,
+  "</s_cnt>": 57529,
+  "</s_creditcardprice>": 57563,
+  "</s_discount_price>": 57557,
+  "</s_discountprice>": 57567,
+  "</s_emoneyprice>": 57569,
+  "</s_etc>": 57541,
+  "</s_itemsubtotal>": 57577,
+  "</s_menu>": 57525,
+  "</s_menuqty_cnt>": 57555,
+  "</s_menutype_cnt>": 57553,
+  "</s_nm>": 57527,
+  "</s_num>": 57565,
+  "</s_othersvc_price>": 57573,
+  "</s_price>": 57531,
+  "</s_service_price>": 57537,
+  "</s_sub>": 57547,
+  "</s_sub_total>": 57533,
+  "</s_subtotal_price>": 57535,
+  "</s_tax_price>": 57539,
+  "</s_total>": 57543,
+  "</s_total_etc>": 57561,
+  "</s_total_price>": 57545,
+  "</s_unitprice>": 57559,
+  "</s_vatyn>": 57575,
+  "</s_void_menu>": 57571,
+  "<s_cashprice>": 57550,
+  "<s_changeprice>": 57552,
+  "<s_cnt>": 57530,
+  "<s_cord-v2>": 57579,
+  "<s_creditcardprice>": 57564,
+  "<s_discount_price>": 57558,
+  "<s_discountprice>": 57568,
+  "<s_emoneyprice>": 57570,
+  "<s_etc>": 57542,
+  "<s_iitcdip>": 57523,
+  "<s_itemsubtotal>": 57578,
+  "<s_menu>": 57526,
+  "<s_menuqty_cnt>": 57556,
+  "<s_menutype_cnt>": 57554,
+  "<s_nm>": 57528,
+  "<s_num>": 57566,
+  "<s_othersvc_price>": 57574,
+  "<s_price>": 57532,
+  "<s_service_price>": 57538,
+  "<s_sub>": 57548,
+  "<s_sub_total>": 57534,
+  "<s_subtotal_price>": 57536,
+  "<s_synthdog>": 57524,
+  "<s_tax_price>": 57540,
+  "<s_total>": 57544,
+  "<s_total_etc>": 57562,
+  "<s_total_price>": 57546,
+  "<s_unitprice>": 57560,
+  "<s_vatyn>": 57576,
+  "<s_void_menu>": 57572,
+  "<sep/>": 57522
+}

config.json

@@ -0,0 +1,187 @@
+{
+  "architectures": [
+    "VisionEncoderDecoderModel"
+  ],
+  "decoder": {
+    "_name_or_path": "",
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "add_cross_attention": true,
+    "add_final_layer_norm": true,
+    "architectures": null,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "bos_token_id": 0,
+    "chunk_size_feed_forward": 0,
+    "classifier_dropout": 0.0,
+    "cross_attention_hidden_size": null,
+    "d_model": 1024,
+    "decoder_attention_heads": 16,
+    "decoder_ffn_dim": 4096,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 4,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "dropout": 0.1,
+    "early_stopping": false,
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 12,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": 2,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "init_std": 0.02,
+    "is_decoder": true,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 768,
+    "min_length": 0,
+    "model_type": "mbart",
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 12,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 1,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "scale_embedding": true,
+    "sep_token_id": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "transformers_version": "4.22.0.dev0",
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "vocab_size": 57580
+  },
+  "encoder": {
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "attention_probs_dropout_prob": 0.0,
+    "bad_words_ids": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "depths": [
+      2,
+      2,
+      14,
+      2
+    ],
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "drop_path_rate": 0.1,
+    "early_stopping": false,
+    "embed_dim": 128,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.0,
+    "hidden_size": 1024,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": [
+      1280,
+      960
+    ],
+    "initializer_range": 0.02,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-05,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "mlp_ratio": 4.0,
+    "model_type": "donut-swin",
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_channels": 3,
+    "num_heads": [
+      4,
+      8,
+      16,
+      32
+    ],
+    "num_layers": 4,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 4,
+    "path_norm": true,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "qkv_bias": true,
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "transformers_version": "4.22.0.dev0",
+    "typical_p": 1.0,
+    "use_absolute_embeddings": false,
+    "use_bfloat16": false,
+    "window_size": 10
+  },
+  "is_encoder_decoder": true,
+  "model_type": "vision-encoder-decoder",
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": null
+}

create_handler.ipynb

@@ -0,0 +1,167 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install transformers --upgrade"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create Custom Handler for Inference Endpoints\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Overwriting handler.py\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%writefile handler.py\n",
+    "from typing import Dict, List, Any\n",
+    "from transformers import DonutProcessor, VisionEncoderDecoderModel\n",
+    "import torch\n",
+    "\n",
+    "\n",
+    "# check for GPU\n",
+    "device = 0 if torch.cuda.is_available() else -1\n",
+    "\n",
+    "\n",
+    "class EndpointHandler:\n",
+    "    def __init__(self, path=\"\"):\n",
+    "        # load the model\n",
+    "        self.processor = DonutProcessor.from_pretrained(path)\n",
+    "        self.model = VisionEncoderDecoderModel.from_pretrained(path)\n",
+    "        # move model to device\n",
+    "        self.model.to(device)\n",
+    "        self.decoder_input_ids = self.processor.tokenizer(\n",
+    "            \"<s_cord-v2>\", add_special_tokens=False, return_tensors=\"pt\"\n",
+    "        ).input_ids\n",
+    "\n",
+    "    def __call__(self, data: Any) -> List[List[Dict[str, float]]]:\n",
+    "\n",
+    "        inputs = data.pop(\"inputs\", data)\n",
+    "\n",
+    "\n",
+    "        # preprocess the input\n",
+    "        pixel_values = self.processor(inputs, return_tensors=\"pt\").pixel_values\n",
+    "\n",
+    "        # forward pass\n",
+    "        outputs = self.model.generate(\n",
+    "            pixel_values.to(device),\n",
+    "            decoder_input_ids=self.decoder_input_ids.to(device),\n",
+    "            max_length=self.model.decoder.config.max_position_embeddings,\n",
+    "            early_stopping=True,\n",
+    "            pad_token_id=self.processor.tokenizer.pad_token_id,\n",
+    "            eos_token_id=self.processor.tokenizer.eos_token_id,\n",
+    "            use_cache=True,\n",
+    "            num_beams=1,\n",
+    "            bad_words_ids=[[self.processor.tokenizer.unk_token_id]],\n",
+    "            return_dict_in_generate=True,\n",
+    "        )\n",
+    "        # process output\n",
+    "        prediction = self.processor.batch_decode(outputs.sequences)[0]\n",
+    "        prediction = self.processor.token2json(prediction)\n",
+    "\n",
+    "        return prediction\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "test custom pipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from handler import EndpointHandler\n",
+    "\n",
+    "my_handler = EndpointHandler(\".\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'menu': [{'nm': '0571-1854 BLUS WANITA',\n",
+       "   'unitprice': '@120.000',\n",
+       "   'cnt': '1',\n",
+       "   'price': '120,000'},\n",
+       "  {'nm': '1002-0060 SHOPPING BAG', 'cnt': '1', 'price': '0'}],\n",
+       " 'total': {'total_price': '120,000',\n",
+       "  'changeprice': '0',\n",
+       "  'creditcardprice': '120,000',\n",
+       "  'menuqty_cnt': '1'}}"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from PIL import Image\n",
+    "\n",
+    "payload = {\"inputs\": Image.open(\"sample.png\").convert(\"RGB\")}\n",
+    "\n",
+    "my_handler(payload)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.9.13 ('dev': conda)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.13"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "f6dd96c16031089903d5a31ec148b80aeb0d39c32affb1a1080393235fbfa2fc"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

handler.py

@@ -0,0 +1,46 @@
+from typing import Dict, List, Any
+from transformers import DonutProcessor, VisionEncoderDecoderModel
+import torch
+
+
+# check for GPU
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+class EndpointHandler:
+    def __init__(self, path=""):
+        # load the model
+        self.processor = DonutProcessor.from_pretrained(path)
+        self.model = VisionEncoderDecoderModel.from_pretrained(path)
+        # move model to device
+        self.model.to(device)
+        self.decoder_input_ids = self.processor.tokenizer(
+            "<s_cord-v2>", add_special_tokens=False, return_tensors="pt"
+        ).input_ids
+
+    def __call__(self, data: Any) -> List[List[Dict[str, float]]]:
+
+        inputs = data.pop("inputs", data)
+
+
+        # preprocess the input
+        pixel_values = self.processor(inputs, return_tensors="pt").pixel_values
+
+        # forward pass
+        outputs = self.model.generate(
+            pixel_values.to(device),
+            decoder_input_ids=self.decoder_input_ids.to(device),
+            max_length=self.model.decoder.config.max_position_embeddings,
+            early_stopping=True,
+            pad_token_id=self.processor.tokenizer.pad_token_id,
+            eos_token_id=self.processor.tokenizer.eos_token_id,
+            use_cache=True,
+            num_beams=1,
+            bad_words_ids=[[self.processor.tokenizer.unk_token_id]],
+            return_dict_in_generate=True,
+        )
+        # process output
+        prediction = self.processor.batch_decode(outputs.sequences)[0]
+        prediction = self.processor.token2json(prediction)
+
+        return prediction

preprocessor_config.json

@@ -0,0 +1,24 @@
+{
+  "do_align_long_axis": false,
+  "do_normalize": true,
+  "do_pad": true,
+  "do_resize": true,
+  "do_thumbnail": true,
+  "feature_extractor_type": "DonutFeatureExtractor",
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "processor_class": "DonutProcessor",
+  "resample": 2,
+  "size": [
+    960,
+    1280
+  ]
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31b78e3d3891072de8e2bf3553b71782242a1f3b589b914ec2b03feff7b14c54
+size 806248251

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb9e3dce4c326195d08fc3dd0f7e2eee1da8595c847bf4c1a9c78b7a82d47e2d
+size 1296245

special_tokens_map.json

@@ -0,0 +1,18 @@
+{
+  "additional_special_tokens": [
+    "<s_cord-v2>"
+  ],
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,22 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "from_slow": true,
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "name_or_path": "naver-clova-ix/donut-base-finetuned-cord-v2",
+  "pad_token": "<pad>",
+  "processor_class": "DonutProcessor",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "special_tokens_map_file": null,
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}