|
|
import requests
|
|
|
import json
|
|
|
import re
|
|
|
from lxml import etree
|
|
|
import chardet
|
|
|
import jieba.analyse
|
|
|
from text2vec import *
|
|
|
from config import Config
|
|
|
|
|
|
def search_bing(query):
|
|
|
"""利用newbing搜索接口，用于检索与query相关的背景信息，作为检索内容
|
|
|
input：query
|
|
|
output：{'url':'','text':'','title':''}
|
|
|
"""
|
|
|
headers = {
|
|
|
'Cookie': 'MUID=2CFCFC26663D64393955ED1C623D62A4; MUIDB=2CFCFC26663D64393955ED1C623D62A4; SRCHD=AF=S00028; SRCHUID=V=2&GUID=76DC1CA8309043BBAB81CFC4C47D76DD&dmnchg=1; _UR=QS=0&TQS=0; MicrosoftApplicationsTelemetryDeviceId=64c1979f-ee59-40a7-928e-b755865bc6ae; ABDEF=V=13&ABDV=13&MRNB=1696643925014&MRB=0; ANON=A=15BC3EC2F3AC041DAD2C715CFFFFFFFF&E=1d05&W=2; NAP=V=1.9&E=1cab&C=MnJiRko1YRJfqV6H22giKijH0-4G1Ub50-Cg7gnMPMN4QFF_OeDZsQ&W=2; PPLState=1; _HPVN=CS=eyJQbiI6eyJDbiI6NiwiU3QiOjAsIlFzIjowLCJQcm9kIjoiUCJ9LCJTYyI6eyJDbiI6NiwiU3QiOjAsIlFzIjowLCJQcm9kIjoiSCJ9LCJReiI6eyJDbiI6NiwiU3QiOjAsIlFzIjowLCJQcm9kIjoiVCJ9LCJBcCI6dHJ1ZSwiTXV0ZSI6dHJ1ZSwiTGFkIjoiMjAyMy0xMi0xMVQwMDowMDowMFoiLCJJb3RkIjowLCJHd2IiOjAsIlRucyI6MCwiRGZ0IjpudWxsLCJNdnMiOjAsIkZsdCI6MCwiSW1wIjoxNiwiVG9iYnMiOjB9; _EDGE_S=SID=2E9C7DC6F71A6094195D6E28F6C8614B; USRLOC=HS=1&ELOC=LAT=30.26828956604004|LON=120.13351440429688|N=%E8%A5%BF%E6%B9%96%E5%8C%BA%EF%BC%8C%E6%B5%99%E6%B1%9F%E7%9C%81|ELT=4|; _Rwho=u=d; _SS=SID=2E9C7DC6F71A6094195D6E28F6C8614B&R=64&RB=64&GB=0&RG=0&RP=61; KievRPSSecAuth=FABKBBRaTOJILtFsMkpLVWSG6AN6C/svRwNmAAAEgAAACFAPF0afImbrCASVm1xT1K+FiXTsQgaoq6RydL+Ge3FvFrIbHVbXa7m0OlZNQJT4P62pu6xUtDTqwGPUE13tWBwVPkK1RahHVaGuUSLfwbp5o2HeLnKa+hfc6+sJiYHnxklhiJAzdi/oBbiWdDkf+5A+C0Fbsxeo4pQDt+kmeKhWpMwijA0bVP5ISXdkrLsRv5jiq97srkAMWFHqqGboI70LdX7ahqSSiykzwFdA1np3WhYhURWQ4b3z6uV7nsZpth6lpdafGZ2YLWr0Zwpv1D210P04ovzbbzvuKGoeljS4/SvdX8QUoGONzn0f2OXAOPvsnZJctbwxH/tkembDlpN4liJDCYhlYgoKtg5nuLBNihk75VctLodAQhosDNYM9stJRzQlusK+aEbDQKAgXunPwB0iPq0ECEVmLIApOeXs7DEtj29Q8zuWiOmxXnddGDm4Tf0VWUVjAEfP/PKiiTLAAS/dwPgOslgEdpy3Pw6GQYo3z3dZ16mWuXYX53utgdkK4rtqRj/FmYiTRjL6scm7Ds0UJnVNxdJcFACadTOzNVEGBp2XIb6XEAWZThz21+JJCn325RXG+zwJyjaKI941n6CbQ8Z/dXgUYMBsn/gfdGV3/+vz05pIOtB1zmzkvwds5v4M/zTcf5fgqWwLjSbIBFscYA626llQwDS6LkKwyoB/EB3L0XgLnOFpoSSpk41L/q5e0GkLVxzPA5kZue0iLTNEXUu/KCGnPOBkK0iAZVS/bJPVa3ZBPBOODwXnAUR0s0W1hbHLDW4I1ZrMuujx08DU0/nhhiq0mFgwwxHrd4vE9xdecjlpyL78pzPf5LVAiCKaWZ/BnKqHCYHA4hiEg8ffC5eFwoA6JsL0wtvTSdaAPEcUs103Um9eje8nNKwvDtesqh93lOAbNCfkfC/zAdtsR0dWaZIsYdAeMNQE+6//zLDbGIe24WVsSdiwZqdmYI2ICxE+KqPY++Ei4gfgKt0GNyiAfK0qSfALb01POx95rWOyWSPd0ge6DwM5mHAZfTePR44vBfFdhvUYBg0+47nOzY53hcO/6unDb3u1/PLHM7+rlS+76yjrZ9Dl7cFXRNBINy5afDUY+kU9JQS6QTbc5EIQTynlWkGU38m43GtWXquitzrjHuC0mYsUbLQOuZ1kFWHQXF/4I/aaxU1k0uvEOttkIUkhXi5lKo9uLoPGdha+AIGcDz8icpdDnfAHHpChm0YB8K8lcL0foY6NCib+o+LCLfriZg9Nvtkc8s1+TWPvCvHZX4bZuXyN4tHoQiysRd6j0gyJpLR4yQr5iOyBUgkM9WWKzkFmnzVYlb4ec6wpowsw2643AHs5Ge1FDjzKw3TdSVnwB2dHFh7tdNW1ywYDAGhpv8SSvQ66448UANVqB1uKwxsD0mXJR/tjMy9OuiNR; _U=1S7ve-XVb_pOh5Iig5kQlQDI6wv9BNl9HiCEtz0dS6dNV_UrQUBmAFVEZx7pYNRTwRxGG8eASH_IDUlpJu04SCp8aeYlPHkU_-0xGzlVA3nTqaE9kSUyIm1UVQYovjbOrsh4SeBbU-wrjqz6HV2DeUKJiHyTwYlDeQ8bYboyqhB4-ER5PjMGcp8daGbur9ER2KSm-nJOeUqnWeIawk0BVyw; WLS=C=26d7831f7a57e7fd&N=; SRCHUSR=DOB=20220207&T=1703137884000&TPC=1703137888000&POEX=W; SRCHHPGUSR=SRCHLANG=zh-Hans&BZA=0&BRW=HTP&BRH=M&CW=975&CH=931&SW=1920&SH=1080&DPR=1&UTC=480&DM=0&HV=1703137888&WTS=63838734684&PRVCW=975&PRVCH=931&SCW=1164&SCH=2821&PV=10.0.0; WLID=mA9cZwKIoBbQ9h07pbmfeYJEn7iBxd5sk7A9mKFJf1dP4SWmtri4X9d1xcl06hKEVmEpT+5GB21NeHYv/uk3maNbHalTEB+UwCwfS7RdzoQ=; _RwBf=r=0&ilt=1&ihpd=1&ispd=0&rc=64&rb=64&gb=0&rg=0&pc=61&mtu=0&rbb=0.0&g=0&cid=&clo=0&v=1&l=2023-12-20T08:00:00.0000000Z&lft=0001-01-01T00:00:00.0000000&aof=0&o=0&p=bingcopilotwaitlist&c=MY00IA&t=3001&s=2023-03-20T09:14:27.6545941+00:00&ts=2023-12-21T05:51:26.7082437+00:00&rwred=0&wls=2&wlb=0&lka=0&lkt=0&aad=0&TH=&mta=0&e=CS-LRz6MT6YjyZDqHmn2zXGq0iVnD2Plg7iI7uA3t-iwF4TTPdW2rejPh5N_c6syhuNr1-uNgqm8vKVLqjaaig&A=15BC3EC2F3AC041DAD2C715CFFFFFFFF&wle=1&ccp=0&ard=0001-01-01T00:00:00.0000000; ipv6=hit=1703141490169&t=4',
|
|
|
'Accept-Encoding': 'gzip, deflate',
|
|
|
'Accept-Language': 'zh-CN,zh;q=0.9',
|
|
|
'Accept': '*/*',
|
|
|
'Referer': 'https://cn.bing.com/search?',
|
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36'
|
|
|
}
|
|
|
|
|
|
res = []
|
|
|
url = 'https://cn.bing.com/search?q=' + query + '&qs=n&form=QBRE'
|
|
|
r = requests.get(url, headers=headers)
|
|
|
try:
|
|
|
encoding = chardet.detect(r.content)['encoding']
|
|
|
r.encoding = encoding
|
|
|
dom = etree.HTML(r.content.decode(encoding))
|
|
|
except:
|
|
|
dom = etree.HTML(r.content)
|
|
|
|
|
|
url_list = []
|
|
|
tmp_url = []
|
|
|
|
|
|
for sel in dom.xpath('//ol[@id="b_results"]/li/h2'):
|
|
|
l = ''.join(sel.xpath('a/@href'))
|
|
|
title = ''.join(sel.xpath('a//text()')).split('-')[0].strip()
|
|
|
if 'http' in l and l not in tmp_url and 'doc.' not in l:
|
|
|
url_list.append([l,title])
|
|
|
tmp_url.append(l)
|
|
|
for turl,title in url_list:
|
|
|
try:
|
|
|
tr = requests.get(turl, headers=headers, timeout=(5, 5))
|
|
|
tdom = etree.HTML(tr.content.decode('utf-8'))
|
|
|
text = '\n'.join(tdom.xpath('//p/text()'))
|
|
|
if len(text) > 15:
|
|
|
tmp = {}
|
|
|
tmp['url'] = turl
|
|
|
tmp['text'] = text
|
|
|
tmp['title'] = title
|
|
|
res.append(tmp)
|
|
|
except Exception as e:
|
|
|
print(e)
|
|
|
pass
|
|
|
return res
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TextRecallRank():
|
|
|
"""
|
|
|
实现对检索内容的召回与排序
|
|
|
"""
|
|
|
|
|
|
def __init__(self,cfg):
|
|
|
self.topk = cfg.topk
|
|
|
self.topd = cfg.topd
|
|
|
self.topt = cfg.topt
|
|
|
self.maxlen = cfg.maxlen
|
|
|
self.recall_way = cfg.recall_way
|
|
|
|
|
|
|
|
|
|
|
|
def query_analyze(self,query):
|
|
|
"""query的解析，目前利用jieba进行关键词提取
|
|
|
input:query,topk
|
|
|
output:
|
|
|
keywords:{'word':[]}
|
|
|
total_weight: float number
|
|
|
"""
|
|
|
keywords = jieba.analyse.extract_tags(query, topK=self.topk, withWeight=True)
|
|
|
total_weight = self.topk / sum([r[1] for r in keywords])
|
|
|
return keywords,total_weight
|
|
|
|
|
|
def text_segmentate(self, text, maxlen, seps='\n', strips=None):
|
|
|
"""将文本按照标点符号划分为若干个短句
|
|
|
"""
|
|
|
text = text.strip().strip(strips)
|
|
|
if seps and len(text) > maxlen:
|
|
|
pieces = text.split(seps[0])
|
|
|
text, texts = '', []
|
|
|
for i, p in enumerate(pieces):
|
|
|
if text and p and len(text) + len(p) > maxlen - 1:
|
|
|
texts.extend(self.text_segmentate(text, maxlen, seps[1:], strips))
|
|
|
text = ''
|
|
|
if i + 1 == len(pieces):
|
|
|
text = text + p
|
|
|
else:
|
|
|
text = text + p + seps[0]
|
|
|
if text:
|
|
|
texts.extend(self.text_segmentate(text, maxlen, seps[1:], strips))
|
|
|
return texts
|
|
|
else:
|
|
|
return [text]
|
|
|
|
|
|
def recall_title_score(self,title,keywords,total_weight):
|
|
|
"""计算query与标题的匹配度"""
|
|
|
score = 0
|
|
|
for item in keywords:
|
|
|
kw, weight = item
|
|
|
if kw in title:
|
|
|
score += round(weight * total_weight,4)
|
|
|
return score
|
|
|
|
|
|
def recall_text_score(self, text, keywords, total_weight):
|
|
|
"""计算query与text的匹配程度"""
|
|
|
score = 0
|
|
|
for item in keywords:
|
|
|
kw, weight = item
|
|
|
p11 = re.compile('%s' % kw)
|
|
|
pr = p11.findall(text)
|
|
|
|
|
|
score += round(weight * total_weight, 4)
|
|
|
return score
|
|
|
|
|
|
def rank_text_by_keywords(self,query,data):
|
|
|
"""通过关键词进行召回"""
|
|
|
|
|
|
|
|
|
keywords,total_weight = self.query_analyze(query)
|
|
|
|
|
|
|
|
|
title_score = {}
|
|
|
for line in data:
|
|
|
title = line['title']
|
|
|
title_score[title] = self.recall_title_score(title,keywords,total_weight)
|
|
|
title_score = sorted(title_score.items(),key=lambda x:x[1],reverse=True)
|
|
|
|
|
|
recall_title_list = [t[0] for t in title_score[:self.topd]]
|
|
|
|
|
|
|
|
|
sentence_score = {}
|
|
|
for line in data:
|
|
|
title = line['title']
|
|
|
text = line['text']
|
|
|
if title in recall_title_list:
|
|
|
for ct in self.text_segmentate(text,self.maxlen, seps='\n。'):
|
|
|
ct = re.sub('\s+', ' ', ct)
|
|
|
if len(ct)>=20:
|
|
|
sentence_score[ct] = self.recall_text_score(ct,keywords,total_weight)
|
|
|
|
|
|
sentence_score = sorted(sentence_score.items(),key=lambda x:x[1],reverse=True)
|
|
|
recall_sentence_list = [s[0] for s in sentence_score[:self.topt]]
|
|
|
return '\n'.join(recall_sentence_list)
|
|
|
|
|
|
def rank_text_by_text2vec(self, query, data):
|
|
|
"""通过text2vec召回"""
|
|
|
if not data:
|
|
|
print("Warning: No data provided for ranking")
|
|
|
return ""
|
|
|
|
|
|
|
|
|
title_list = [query]
|
|
|
for line in data:
|
|
|
title = line['title']
|
|
|
title_list.append(title)
|
|
|
|
|
|
|
|
|
if len(title_list) <= 1:
|
|
|
print("Warning: Not enough titles for similarity calculation")
|
|
|
return ""
|
|
|
|
|
|
title_vectors = get_vector(title_list, 8)
|
|
|
|
|
|
|
|
|
if title_vectors.numel() == 0 or title_vectors.size(0) <= 1:
|
|
|
print("Warning: Title vectorization failed or returned insufficient vectors")
|
|
|
return ""
|
|
|
|
|
|
title_score = get_sim(title_vectors)
|
|
|
|
|
|
|
|
|
if not title_score:
|
|
|
print("Warning: Title similarity calculation failed")
|
|
|
return ""
|
|
|
|
|
|
title_score = dict(zip(title_score, range(1, len(title_list))))
|
|
|
title_score = sorted(title_score.items(), key=lambda x:x[0], reverse=True)
|
|
|
|
|
|
|
|
|
if not title_score or self.topd <= 0:
|
|
|
print("Warning: No title scores or invalid topd parameter")
|
|
|
return ""
|
|
|
|
|
|
recall_title_list = [title_list[t[1]] for t in title_score[:min(self.topd, len(title_score))]]
|
|
|
|
|
|
|
|
|
sentence_list = [query]
|
|
|
for line in data:
|
|
|
title = line['title']
|
|
|
text = line['text']
|
|
|
if title in recall_title_list:
|
|
|
for ct in self.text_segmentate(text, self.maxlen, seps='\n。'):
|
|
|
ct = re.sub('\s+', ' ', ct)
|
|
|
if len(ct) >= 20:
|
|
|
sentence_list.append(ct)
|
|
|
|
|
|
|
|
|
if len(sentence_list) <= 1:
|
|
|
print("Warning: Not enough sentences for similarity calculation")
|
|
|
return ""
|
|
|
|
|
|
sentence_vectors = get_vector(sentence_list, 8)
|
|
|
|
|
|
|
|
|
if sentence_vectors.numel() == 0 or sentence_vectors.size(0) <= 1:
|
|
|
print("Warning: Sentence vectorization failed or returned insufficient vectors")
|
|
|
return ""
|
|
|
|
|
|
sentence_score = get_sim(sentence_vectors)
|
|
|
|
|
|
|
|
|
if not sentence_score:
|
|
|
print("Warning: Sentence similarity calculation failed")
|
|
|
return ""
|
|
|
|
|
|
sentence_score = dict(zip(sentence_score, range(1, len(sentence_list))))
|
|
|
sentence_score = sorted(sentence_score.items(), key=lambda x:x[0], reverse=True)
|
|
|
|
|
|
|
|
|
if not sentence_score or self.topt <= 0:
|
|
|
print("Warning: No sentence scores or invalid topt parameter")
|
|
|
return ""
|
|
|
|
|
|
recall_sentence_list = [sentence_list[s[1]] for s in sentence_score[:min(self.topt, len(sentence_score))]]
|
|
|
return '\n'.join(recall_sentence_list)
|
|
|
|
|
|
|
|
|
def query_retrieve(self,query):
|
|
|
|
|
|
data = search_bing(query)
|
|
|
|
|
|
if self.recall_way == 'keyword':
|
|
|
bg_text = self.rank_text_by_keywords(query,data)
|
|
|
else:
|
|
|
bg_text = self.rank_text_by_text2vec(query,data)
|
|
|
return bg_text
|
|
|
|
|
|
|
|
|
cfg = Config()
|
|
|
trr = TextRecallRank(cfg)
|
|
|
q_searching = trr.query_retrieve
|
|
|
|
