Duplicate from zhangj726/poem_generation

Co-authored-by: Jing Zhang <[email protected]>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,22 @@
+# PyTorch
+/.torch
+
+# Data files
+*.csv
+*.json
+*.tsv
+
+# Model files
+*.ckpt
+*.pth
+*.pkl
+
+# Logs and checkpoints
+logs/
+checkpoints/
+
+# Secondary files
+*.pyc
+__pycache__/
+.DS_Store
+

.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

.idea/.name

@@ -0,0 +1 @@
+inference.py

.idea/ea_lstm.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,20 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="7">
+            <item index="0" class="java.lang.String" itemvalue="easydict" />
+            <item index="1" class="java.lang.String" itemvalue="pandas" />
+            <item index="2" class="java.lang.String" itemvalue="matplotlib" />
+            <item index="3" class="java.lang.String" itemvalue="pillow" />
+            <item index="4" class="java.lang.String" itemvalue="mindspore" />
+            <item index="5" class="java.lang.String" itemvalue="setuptools" />
+            <item index="6" class="java.lang.String" itemvalue="numpy" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (pytorch)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/ea_lstm.iml" filepath="$PROJECT_DIR$/.idea/ea_lstm.iml" />
+    </modules>
+  </component>
+</project>

.idea/nlp.iml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="PyDocumentationSettings">
+    <option name="format" value="PLAIN" />
+    <option name="myDocStringFormat" value="Plain" />
+  </component>
+</module>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

.idea/workspace.xml

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ChangeListManager">
+    <list default="true" id="276a53df-3cdd-4e96-95d3-c1e69d4e9b9f" name="Changes" comment="" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="MarkdownSettingsMigration">
+    <option name="stateVersion" value="1" />
+  </component>
+  <component name="ProjectId" id="2OyFWrJQpFYHFKgf87OgmRH5Jtu" />
+  <component name="ProjectViewState">
+    <option name="hideEmptyMiddlePackages" value="true" />
+    <option name="showLibraryContents" value="true" />
+  </component>
+  <component name="PropertiesComponent"><![CDATA[{
+  "keyToString": {
+    "RunOnceActivity.OpenProjectViewOnStart": "true",
+    "RunOnceActivity.ShowReadmeOnStart": "true",
+    "last_opened_file_path": "C:/Users/LENOVO/PycharmProjects/lstm"
+  }
+}]]></component>
+  <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="276a53df-3cdd-4e96-95d3-c1e69d4e9b9f" name="Changes" comment="" />
+      <created>1682524950142</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1682524950142</updated>
+    </task>
+    <servers />
+  </component>
+  <component name="XDebuggerManager">
+    <watches-manager>
+      <configuration name="PythonConfigurationType">
+        <watch expression="input_eval" />
+        <watch expression="word_2_index" />
+      </configuration>
+    </watches-manager>
+  </component>
+</project>

README.md

@@ -0,0 +1,26 @@
+---
+duplicated_from: zhangj726/poem_generation
+---
+# NLP Final Project
+
+```shell
+├── configs
+├── data
+│   └── poetry.txt
+├── inference.py
+├── src
+│   ├── apis
+│   │   ├── evaluate.py
+│   │   ├── inference.py
+│   │   └── train.py
+│   ├── datasets
+│   │   └── dataloader.py
+│   ├── models
+│   │   └── EA-LSTM
+│   │       ├── algorithm.py
+│   │       └── model.py
+│   └── utils
+│       └── utils.py
+├── test.py
+└── train.py
+```

app.py

@@ -0,0 +1,14 @@
+# !/user/bin/env python3
+# -*- coding: utf-8 -*-
+
+import gradio
+from inference import infer
+
+
+
+
+INTERFACE = gradio.Interface(fn=infer, inputs=[gradio.Radio(["lstm","GRU"]),"text"], outputs=["text"], title="Poetry Generation",
+                             description="Choose a model and input the poetic head to generate a acrostic",
+                             thumbnail="https://github.com/gradio-app/gpt-2/raw/master/screenshots/interface.png?raw=true")
+
+INTERFACE.launch(inbrowser=True)

data/word_vec.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1164cfc2e28ef6ecbb1a04734e7268238b4841667f13d6cb4c42e27717dd4575
+size 6339344

inference.py

@@ -0,0 +1,108 @@
+import torch
+import argparse
+import numpy as np
+from src.models.LSTM.model import Poetry_Model_lstm
+from src.datasets.dataloader import train_vec
+from src.utils.utils import make_cuda
+
+
+def parse_arguments():
+    # argument parsing
+    parser = argparse.ArgumentParser(description="Specify Params for Experimental Setting")
+    parser.add_argument('--model', type=str, default='lstm',
+                        help="lstm/GRU/Seq2Seq/Transformer/GPT-2")
+    parser.add_argument('--Word2Vec', default=True)
+    parser.add_argument('--strict_dataset', default=False, help="strict dataset")
+    parser.add_argument('--n_hidden', type=int, default=128)
+
+    parser.add_argument('--save_path', type=str, default='save_models/lstm_50.pth')
+
+    return parser.parse_args()
+
+
+def generate_poetry(model, head_string, w1, word_2_index, index_2_word):
+    print("藏头诗生成中...., {}".format(head_string))
+    poem = ""
+    # 以句子的每一个字为开头生成诗句
+    for head in head_string:
+        if head not in word_2_index:
+            print("抱歉，不能生成以{}开头的诗".format(head))
+            return
+
+        sentence = head
+        max_sent_len = 20
+
+        h_0 = torch.tensor(np.zeros((2, 1, args.n_hidden), dtype=np.float32))
+        c_0 = torch.tensor(np.zeros((2, 1, args.n_hidden), dtype=np.float32))
+
+        input_eval = word_2_index[head]
+        for i in range(max_sent_len):
+            if args.Word2Vec:
+                word_embedding = torch.tensor(w1[input_eval][None][None])
+            else:
+                word_embedding = torch.tensor([input_eval]).unsqueeze(dim=0)
+            pre, (h_0, c_0) = model(word_embedding, h_0, c_0)
+            char_generated = index_2_word[int(torch.argmax(pre))]
+
+            if char_generated == '。':
+                break
+            # 以新生成的字为输入继续向下生成
+            input_eval = word_2_index[char_generated]
+            sentence += char_generated
+
+        poem += '\n' + sentence
+
+    return poem
+
+def infer(model,string):
+    args = parse_arguments()
+    all_data, (w1, word_2_index, index_2_word) = train_vec()
+    args.word_size, args.embedding_num = w1.shape
+    # string = input("诗头:")
+    # string = '自然语言'
+    args.model=model
+    if args.model == 'lstm':
+        model = Poetry_Model_lstm(args.n_hidden, args.word_size, args.embedding_num, args.Word2Vec)
+        args.save_path = 'save_models/lstm_50.pth'
+    elif args.model == 'GRU':
+        model = Poetry_Model_lstm(args.n_hidden, args.word_size, args.embedding_num, args.Word2Vec)
+        args.save_path = 'save_models/GRU_50.pth'
+    elif args.model == 'Seq2Seq':
+        model = Poetry_Model_lstm(args.n_hidden, args.word_size, args.embedding_num, args.Word2Vec)
+    elif args.model == 'Transformer':
+        model = Poetry_Model_lstm(args.n_hidden, args.word_size, args.embedding_num, args.Word2Vec)
+    elif args.model == 'GPT-2':
+        model = Poetry_Model_lstm(args.n_hidden, args.word_size, args.embedding_num, args.Word2Vec)
+    else:
+        print("Please choose a model!\n")
+
+    model.load_state_dict(torch.load(args.save_path))
+    model = make_cuda(model)
+    poem = generate_poetry(model, string, w1, word_2_index, index_2_word)
+    return poem
+
+
+if __name__ == '__main__':
+    args = parse_arguments()
+    all_data, (w1, word_2_index, index_2_word) = train_vec()
+    args.word_size, args.embedding_num = w1.shape
+    # string = input("诗头:")
+    string = '自然语言'
+
+    if args.model == 'lstm':
+        model = Poetry_Model_lstm(args.n_hidden, args.word_size, args.embedding_num, args.Word2Vec)
+    elif args.model == 'GRU':
+        model = Poetry_Model_lstm(args.n_hidden, args.word_size, args.embedding_num, args.Word2Vec)
+    elif args.model == 'Seq2Seq':
+        model = Poetry_Model_lstm(args.n_hidden, args.word_size, args.embedding_num, args.Word2Vec)
+    elif args.model == 'Transformer':
+        model = Poetry_Model_lstm(args.n_hidden, args.word_size, args.embedding_num, args.Word2Vec)
+    elif args.model == 'GPT-2':
+        model = Poetry_Model_lstm(args.n_hidden, args.word_size, args.embedding_num, args.Word2Vec)
+    else:
+        print("Please choose a model!\n")
+
+    model.load_state_dict(torch.load(args.save_path))
+    model = make_cuda(model)
+    poem = generate_poetry(model, string, w1, word_2_index, index_2_word)
+    print(poem)

requirements.txt

@@ -0,0 +1,3 @@
+torch==1.13.0
+gradio==3.34.0
+gensim==4.3.1

save_models/GRU_25.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bacf9a7ec329c6185098c1309ab28239b4c087b53832b3d18e5323831bfead23
+size 10727391

save_models/GRU_50.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a8e83a733c023b35c44020e014bb72e2c1d05698eb782669c0e4d5a76d4590d
+size 10727391

save_models/lstm_25.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b064666ce02c63541dee4b6146d31ee8f7e784ee9c2811c9b9266aba6cc4193
+size 10727391

save_models/lstm_50.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa157d970149c32b53b024a23ef8428e7b7e1702ed72d44152b568b085b1bfaa
+size 10727391

save_models/transformer_100.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bbe153237c20ba6ec5e8ac8b55c8b420ec4cdf5bf0f46a8a5b68094a54996c3
+size 26125257

src/apis/evaluate.py

@@ -0,0 +1,23 @@
+import torch
+import numpy as np
+from src.models.EA_LSTM.model import weightedLSTM
+from src.datasets.dataloader import MyDataset, create_vocab
+
+
+def test(args):
+    vocab, poetrys = create_vocab(args.data)
+    # 词汇表长度
+    args.vocab_size = len(vocab)
+    int2char = np.array(vocab)
+    valid_dataset = MyDataset(vocab, poetrys, args, train=False)
+
+    model = weightedLSTM(6110, 256, 128, 2, [1.0] * 80, False)
+    model.load_state_dict(torch.load(args.save_path))
+
+    input_example_batch, target_example_batch = valid_dataset[0]
+    example_batch_predictions = model(input_example_batch)
+    predicted_id = torch.distributions.Categorical(example_batch_predictions).sample()
+    predicted_id = torch.squeeze(predicted_id, -1).numpy()
+    print("Input: \n", repr("".join(int2char[input_example_batch])))
+    print()
+    print("Predictions: \n", repr("".join(int2char[predicted_id])))

src/apis/train.py

@@ -0,0 +1,68 @@
+import math
+import torch
+import numpy as np
+import torch.nn as nn
+import torch.optim as optim
+from src.utils.utils import make_cuda
+from torch.nn import functional as F
+from sklearn.metrics import mean_squared_error, mean_absolute_error
+
+
+def train(args, model, data_loader, initial=False):
+    optimizer = optim.Adam(model.parameters(), lr=args.learning_rate)
+
+    model.train()
+    num_epochs = args.initial_epochs if initial else args.num_epochs
+
+    for epoch in range(num_epochs):
+        loss = 0
+        for step, (features, targets) in enumerate(data_loader):
+            features = make_cuda(features)
+            targets = make_cuda(targets)
+
+            optimizer.zero_grad()
+
+            pre, _ = model(features)
+            crs_loss = model.cross_entropy(pre, targets.reshape(-1))
+            loss += crs_loss.item()
+            crs_loss.backward()
+            torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
+            optimizer.step()
+
+            # print step info
+            if (step + 1) % args.log_step == 0:
+                print("Epoch [%.3d/%.3d] Step [%.3d/%.3d]: CROSS_loss=%.4f, RCROSS_loss=%.4f"
+                      % (epoch + 1,
+                         num_epochs,
+                         step + 1,
+                         len(data_loader),
+                         loss / args.log_step,
+                         math.sqrt(loss / args.log_step)))
+                loss = 0
+
+        # Loss = []
+        # for step, (features, targets) in enumerate(valid_data_loader):
+        #     features = make_cuda(features)
+        #     targets = make_cuda(targets)
+        #     model.eval()
+        #     preds = model(features)
+        #     valid_loss = CrossLoss(preds, targets)
+        #     Loss.append(valid_loss)
+        # print("Valid loss: %.3d\n" % (np.mean(Loss)))
+
+    return model
+
+
+def evaluate(args, model, data_loader):
+    model.eval()
+    loss = []
+    for step, (features, targets) in enumerate(data_loader):
+        features = make_cuda(features)
+        targets = make_cuda(targets)
+
+        pre, _ = model(features)
+        crs_loss = model.cross_entropy(pre, targets.reshape(-1))
+        loss.append(crs_loss.item())
+        torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
+
+    print("loss=%.4f" % (np.mean(loss)))

src/datasets/dataloader.py

@@ -0,0 +1,115 @@
+import numpy as np
+import pickle
+import os
+import torch
+import torch.nn as nn
+from gensim.models.word2vec import Word2Vec
+from torch.utils.data import Dataset
+
+
+def padding(poetries, maxlen, pad):
+    batch_seq = [poetry + pad * (maxlen - len(poetry)) for poetry in poetries]
+    return batch_seq
+
+
+# 输入向后滑一字符为target，即预测下一个字
+def split_input_target(seq):
+    inputs = seq[:-1]
+    targets = seq[1:]
+    return inputs, targets
+
+
+# 创建词汇表
+def get_poetry(arg):
+    poetrys = []
+    if arg.Augmented_dataset:
+        path = arg.Augmented_data
+    else:
+        path = arg.data
+    with open(path, "r", encoding='UTF-8') as f:
+        for line in f:
+            try:
+                # line = line.decode('UTF-8')
+                line = line.strip(u'\n')
+                if arg.Augmented_dataset:
+                    content = line.strip(u' ')
+                else:
+                    title, content = line.strip(u' ').split(u':')
+                content = content.replace(u' ', u'')
+                if u'_' in content or u'(' in content or u'（' in content or u'《' in content or u'[' in content:
+                    continue
+                if arg.strict_dataset:
+                    if len(content) < 12 or len(content) > 79:
+                        continue
+                else:
+                    if len(content) < 5 or len(content) > 79:
+                        continue
+                content = u'[' + content + u']'
+                poetrys.append(content)
+            except Exception as e:
+                pass
+
+            # 按诗的字数排序
+    poetrys = sorted(poetrys, key=lambda line: len(line))
+
+    with open("data/org_poetry.txt", "w", encoding="utf-8") as f:
+        for poetry in poetrys:
+            poetry = str(poetry).strip('[').strip(']').replace(',', '').replace('\'', '') + '\n'
+            f.write(poetry)
+
+    return poetrys
+
+
+# 切分文档
+def split_text(poetrys):
+    with open("data/split_poetry.txt", "w", encoding="utf-8") as f:
+        for poetry in poetrys:
+            poetry = str(poetry).strip('[').strip(']').replace(',', '').replace('\'', '') + '\n '
+            split_data = " ".join(poetry)
+            f.write(split_data)
+    return open("data/split_poetry.txt", "r", encoding='UTF-8').read()
+
+
+# 训练词向量
+def train_vec(split_file="data/split_poetry.txt", org_file="data/org_poetry.txt"):
+    param_file = "data/word_vec.pkl"
+    org_data = open(org_file, "r", encoding="utf-8").read().split("\n")
+    if os.path.exists(split_file):
+        all_data_split = open(split_file, "r", encoding="utf-8").read().split("\n")
+    else:
+        all_data_split = split_text().split("\n")
+
+    if os.path.exists(param_file):
+        return org_data, pickle.load(open(param_file, "rb"))
+
+    models = Word2Vec(all_data_split, vector_size=256, workers=7, min_count=1)
+    pickle.dump([models.syn1neg, models.wv.key_to_index, models.wv.index_to_key], open(param_file, "wb"))
+    return org_data, (models.syn1neg, models.wv.key_to_index, models.wv.index_to_key)
+
+
+class Poetry_Dataset(Dataset):
+    def __init__(self, w1, word_2_index, all_data, Word2Vec):
+        self.Word2Vec = Word2Vec
+        self.w1 = w1
+        self.word_2_index = word_2_index
+        word_size, embedding_num = w1.shape
+        self.embedding = nn.Embedding(word_size, embedding_num)
+        # 最长句子长度
+        maxlen = max([len(seq) for seq in all_data])
+        pad = ' '
+        self.all_data = padding(all_data[:-1], maxlen, pad)
+
+    def __getitem__(self, index):
+        a_poetry = self.all_data[index]
+
+        a_poetry_index = [self.word_2_index[i] for i in a_poetry]
+        xs, ys = split_input_target(a_poetry_index)
+        if self.Word2Vec:
+            xs_embedding = self.w1[xs]
+        else:
+            xs_embedding = np.array(xs)
+
+        return xs_embedding, np.array(ys).astype(np.int64)
+
+    def __len__(self):
+        return len(self.all_data)