init

.gitattributes

@@ -0,0 +1,43 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*.tfevents* filter=lfs diff=lfs merge=lfs -text
+*.db* filter=lfs diff=lfs merge=lfs -text
+*.ark* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
+pytorch_model.bin filter=lfs diff=lfs merge=lfs -text
+resources/.ipynb_checkpoints/ocr_general-checkpoint.png filter=lfs diff=lfs merge=lfs -text
+resources/.ipynb_checkpoints/OFA_logo_tp_path-checkpoint.svg filter=lfs diff=lfs merge=lfs -text
+resources/.ipynb_checkpoints/image_ocr_recognition-checkpoint.jpg filter=lfs diff=lfs merge=lfs -text
+resources/.ipynb_checkpoints/ocr_general_demo-checkpoint.png filter=lfs diff=lfs merge=lfs -text
+resources/.ipynb_checkpoints/ocr_scene-checkpoint.png filter=lfs diff=lfs merge=lfs -text
+resources/OFA_logo_tp_path.svg filter=lfs diff=lfs merge=lfs -text
+resources/image_ocr_recognition.jpg filter=lfs diff=lfs merge=lfs -text
+resources/ocr_general.png filter=lfs diff=lfs merge=lfs -text
+resources/ocr_general_demo.png filter=lfs diff=lfs merge=lfs -text
+resources/ocr_scene.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,209 @@
+---
+backbone:
+- OFA
+domain:
+- multi-modal
+frameworks:
+- pytorch
+license: Apache License 2.0
+metrics:
+- accuracy
+tags:
+- Alibaba
+- ICML2022
+- arxiv:2202.03052
+tasks:
+- ocr-recognition
+
+datasets:
+  evaluation:
+  - modelscope/ocr_fudanvi_zh
+  train:
+  - modelscope/ocr_fudanvi_zh
+finetune-support: True
+integrating: False
+widgets:
+  - task: ofa-ocr-recognition
+    inputs:
+      - name: image
+        title: 图片
+        type: image
+        validator:
+        max_resolution: 5000*5000
+        max_size: 10M
+    examples:
+      - name: 1
+        title: 示例1 
+        inputs:
+        - data: https://xingchen-data.oss-cn-zhangjiakou.aliyuncs.com/maas/ocr/ocr_general_demo.png
+          name: image
+    inferencespec:
+      cpu: 4
+      gpu: 1
+      gpu_memory: 16000
+      memory: 43000
+integrating: True
+---
+# OFA-文字识别
+## News
+- 2023年1月：
+  - 优化了finetune流程，支持参数更新、自定义数据及脚本分布式训练等，见finetune示例。
+- 2022年11月：
+  - 发布ModelScope 1.0版本，以下能力请使用1.0.2及以上版本。
+  - 支持finetune能力，新增[OFA Tutorial](https://www.modelscope.cn/docs/OFA%20Tutorial)，finetune能力参考1.4节。
+
+
+## 文字识别是什么？
+文字识别，即给定一张文本图片，识别出图中所含文字并输出对应字符串，欢迎使用！
+
+
+## 快速玩起来
+玩转OFA只需区区以下6行代码，就是如此轻松！如果你觉得还不够方便，请点击右上角`Notebook`按钮，我们为你提供了配备了GPU的环境，你只需要在notebook里输入提供的代码，就可以把OFA玩起来了！
+
+<p align="center">
+    <img src="resources/ocr_general_demo.png" alt="ocr" width="200" />
+
+```python
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+from modelscope.outputs import OutputKeys
+
+# ModelScope Library >= 1.2.0
+ocr_recognize = pipeline(Tasks.ocr_recognition, model='damo/ofa_ocr-recognition_general_base_zh', model_revision='v1.0.2')
+result = ocr_recognize('https://xingchen-data.oss-cn-zhangjiakou.aliyuncs.com/maas/ocr/ocr_general_demo.png')
+print(result[OutputKeys.TEXT])
+```
+<br>
+
+## OFA是什么？
+OFA(One-For-All)是通用多模态预训练模型，使用简单的序列到序列的学习框架统一模态（跨模态、视觉、语言等模态）和任务（如图片生成、视觉定位、图片描述、图片分类、文本生成等），详见我们发表于ICML 2022的论文：[OFA: Unifying Architectures, Tasks, and Modalities Through a Simple Sequence-to-Sequence Learning Framework](https://arxiv.org/abs/2202.03052)，以及我们的官方Github仓库[https://github.com/OFA-Sys/OFA](https://github.com/OFA-Sys/OFA)。
+
+<p align="center">
+    <br>
+    <img src="resources/OFA_logo_tp_path.svg" width="150" />
+    <br>
+<p>
+<br>
+
+<p align="center">
+        <a href="https://github.com/OFA-Sys/OFA">Github</a>&nbsp ｜ &nbsp<a href="https://arxiv.org/abs/2202.03052">Paper </a>&nbsp ｜ &nbspBlog
+</p>
+
+<p align="center">
+    <br>
+        <video src="https://xingchen-data.oss-cn-zhangjiakou.aliyuncs.com/maas/resources/modelscope_web/demo.mp4" loop="loop" autoplay="autoplay" muted width="100%"></video>
+    <br>
+</p>
+
+
+## 为什么OFA是文字识别的最佳选择？
+OFA在文字识别（ocr recognize）在公开数据集(including RCTW, ReCTS, LSVT, ArT, CTW)中进行评测, 在准确率指标上达到SOTA结果，具体如下：
+<p align="left">
+<table border="1" width="100%">
+    <tr align="center">
+        <td>Model</td><td>Scene</td><td>Web</td><td>Document</td><td>Handwriting</td><td>Avg</td>
+    </tr>
+    <tr align="center">
+        <td>SAR</td><td>62.5</td><td>54.3</td><td>93.8</td><td>31.4</td><td>67.3</td>
+    </tr>
+    <tr align="center">
+        <td>TransOCR</td><td>63.3</td><td>62.3</td><td>96.9</td><td>53.4</td><td>72.8</td>
+    </tr>
+    <tr align="center">
+        <td>MaskOCR-base</td><td>73.9</td><td>74.8</td><td>99.3</td><td>63.7</td><td>80.8</td>
+    </tr>
+    <tr align="center">
+        <td>OFA-OCR</td><td>82.9</td><td>81.7</td><td>99.1</td><td>69.0</td><td>86.0</td>
+    </tr>
+</table>
+<br>
+</p>
+
+## 模型训练流程
+
+### 训练数据介绍
+本模型训练数据集是复旦大学视觉智能实验室，数据链接：https://github.com/FudanVI/benchmarking-chinese-text-recognition
+场景数据集图片采样：
+<p align="center">
+    <img src="./resources/ocr_general.png" width="500" />
+</p>
+
+### 训练流程
+模型及finetune细节请参考[OFA Tutorial](https://modelscope.cn/docs/OFA_Tutorial#1.4%20%E5%A6%82%E4%BD%95%E8%AE%AD%E7%BB%83) 1.4节。
+
+### Finetune示例
+```python
+import tempfile
+from modelscope.msdatasets import MsDataset
+from modelscope.metainfo import Trainers
+from modelscope.trainers import build_trainer
+from modelscope.utils.constant import DownloadMode
+
+train_dataset = MsDataset(MsDataset.load(
+        'ocr_fudanvi_zh',
+        subset_name='scene',
+        namespace='modelscope',
+        split='train[:100]',
+        download_mode=DownloadMode.REUSE_DATASET_IF_EXISTS).remap_columns({
+        'label': 'text'
+    }))
+
+test_dataset = MsDataset(
+    MsDataset.load(
+        'ocr_fudanvi_zh',
+        subset_name='scene',
+        namespace='modelscope',
+        split='test[:20]',
+        download_mode=DownloadMode.REUSE_DATASET_IF_EXISTS).remap_columns({
+        'label': 'text'
+    }))
+
+# 可以在代码修改 configuration 的配置
+def cfg_modify_fn(cfg):
+    cfg.train.hooks = [{
+        'type': 'CheckpointHook',
+        'interval': 2
+    }, {
+        'type': 'TextLoggerHook',
+        'interval': 1
+    }, {
+        'type': 'IterTimerHook'
+    }]
+    cfg.train.max_epochs=2
+    return cfg
+
+args = dict(
+    model='damo/ofa_ocr-recognition_general_base_zh',
+    model_revision='v1.0.2',
+    train_dataset=train_dataset,
+    eval_dataset=test_dataset,
+    cfg_modify_fn=cfg_modify_fn,
+    work_dir = tempfile.TemporaryDirectory().name)
+trainer = build_trainer(name=Trainers.ofa, default_args=args)
+trainer.train()
+```
+
+## 模型局限性以及可能的偏差
+训练数据集自身有局限，有可能产生一些偏差，请用户自行评测后决定如何使用。
+
+## 相关论文以及引用
+如果你觉得OFA好用，喜欢我们的工作，欢迎引用：
+```
+@article{wang2022ofa,
+  author    = {Peng Wang and
+               An Yang and
+               Rui Men and
+               Junyang Lin and
+               Shuai Bai and
+               Zhikang Li and
+               Jianxin Ma and
+               Chang Zhou and
+               Jingren Zhou and
+               Hongxia Yang},
+  title     = {OFA: Unifying Architectures, Tasks, and Modalities Through a Simple Sequence-to-Sequence
+               Learning Framework},
+  journal   = {CoRR},
+  volume    = {abs/2202.03052},
+  year      = {2022}
+}
+```

config.json

@@ -0,0 +1,54 @@
+{
+  "activation_dropout": 0.0,
+  "activation_function": "gelu",
+  "add_type_embedding": true,
+  "architectures": [
+    "OFAModel"
+  ],
+  "attention_dropout": 0.0,
+  "attn_scale_factor": 2.0,
+  "bos_token_id": 0,
+  "classifier_dropout": 0.0,
+  "code_image_size": 128,
+  "code_layernorm_embedding": true,
+  "d_model": 768,
+  "decoder_attention_heads": 12,
+  "decoder_drop_path_rate": 0.0,
+  "decoder_ffn_dim": 3072,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 6,
+  "decoder_normalize_before": true,
+  "decoder_start_token_id": 0,
+  "dropout": 0.1,
+  "encoder_attention_heads": 12,
+  "encoder_drop_path_rate": 0.0,
+  "encoder_ffn_dim": 3072,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 6,
+  "encoder_normalize_before": true,
+  "entangle_position_embedding": false,
+  "eos_token_id": 2,
+  "forced_eos_token_id": 2,
+  "image_bucket_size": 42,
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "layernorm_embedding": true,
+  "max_position_embeddings": 1024,
+  "model_type": "ofa",
+  "normformer": true,
+  "num_hidden_layers": 6,
+  "pad_token_id": 1,
+  "patch_layernorm_embedding": true,
+  "resnet_drop_path_rate": 0.0,
+  "resnet_model_path": null,
+  "resnet_type": "resnet101",
+  "scale_embedding": false,
+  "share_decoder_input_output_embed": true,
+  "token_bucket_size": 256,
+  "torch_dtype": "float32",
+  "transformers_version": "4.22.2",
+  "use_cache": true,
+  "vocab_size": 30325,
+  "interpolate_position": true,
+  "orig_patch_image_size": 224
+}

configuration.json

@@ -0,0 +1,102 @@
+{
+    "framework": "pytorch",
+
+    "task": "ocr-recognition",
+
+    "model": {
+        "type": "ofa",
+        "beam_search": {
+            "beam_size": 5,
+            "max_len_b": 64,
+            "min_len": 1,
+            "no_repeat_ngram_size": 0
+        },
+        "seed": 7,
+        "max_src_length": 128,
+        "language": "zh",
+        "prompt": "图片上的文字是什么?",
+        "gen_type": "generation",
+        "patch_image_size": 480,
+        "max_image_size": 480,
+        "is_document": false,
+        "imagenet_default_mean_and_std": false
+    },
+    "pipeline": {
+        "type": "ofa-ocr-recognition"
+    },
+    "dataset": {
+        "column_map": {
+            "text": "text",
+            "image": "image"
+        }
+    },
+    "train": {
+        "work_dir": "/tmp",
+        "max_epochs": 1,
+        "use_fp16": false,
+        "dataloader": {
+            "batch_size_per_gpu": 4,
+            "workers_per_gpu": 0
+        },
+        "lr_scheduler": {
+            "name": "polynomial_decay",
+            "warmup_proportion": 0.01,
+            "lr_end": 1e-07
+        },
+        "lr_scheduler_hook": {
+            "type": "LrSchedulerHook",
+            "by_epoch": false
+        },
+        "optimizer": {
+            "type": "AdamW",
+            "lr": 5e-05,
+            "weight_decay": 0.01
+        },
+        "optimizer_hook": {
+            "type": "TorchAMPOptimizerHook",
+            "cumulative_iters": 1,
+            "grad_clip": {
+                "max_norm": 1.0,
+                "norm_type": 2
+            },
+            "loss_keys": "loss"
+        },
+        "criterion": {
+            "name": "AdjustLabelSmoothedCrossEntropyCriterion",
+            "constraint_range": null,
+            "drop_worst_after": 0,
+            "drop_worst_ratio": 0.0,
+            "ignore_eos": false,
+            "ignore_prefix_size": 0,
+            "label_smoothing": 0.1,
+            "reg_alpha": 1.0,
+            "report_accuracy": false,
+            "sample_patch_num": 196,
+            "sentence_avg": false,
+            "use_rdrop": true
+        },
+        "hooks": [{
+                "type": "BestCkptSaverHook",
+                "metric_key": "accuracy",
+                "interval": 100
+            },
+            {
+                "type": "TextLoggerHook",
+                "interval": 1
+            },
+            {
+                "type": "IterTimerHook"
+            }
+        ]
+    },
+    "evaluation": {
+        "dataloader": {
+            "batch_size_per_gpu": 4,
+            "workers_per_gpu": 0
+        },
+        "metrics": [{
+            "type": "accuracy"
+        }]
+    },
+    "preprocessor": []
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a39fe1cdce5c98f554a04f7caf76f5be02dfdc1f440611d186bd878fc8b02c19
+size 706781709

tokenizer_config.json

@@ -0,0 +1,3 @@
+{
+  "do_lower_case": false
+}