ok

.gitea/workflows/build.yaml

@@ -0,0 +1,47 @@
+name: Build
+run-name: ${{ github.actor }} is upgrade release 🚀
+on: [push]
+env:
+  REPOSITORY: ${{ github.repository }}
+  COMMIT_ID: ${{ github.sha }}
+jobs:
+  Build-Deploy-Actions:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event."
+      - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!"
+      - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      -
+        name: Setup Git LFS
+        run: |
+          git lfs install
+          git lfs fetch
+          git lfs checkout                    
+      - name: List files in the repository
+        run: |
+                                        ls ${{ github.workspace }}
+      -
+        name: Docker Image Info
+        id: image-info
+        run: |
+          echo "::set-output name=image_name::$(echo $REPOSITORY | tr '[:upper:]' '[:lower:]')"
+          echo "::set-output name=image_tag::${COMMIT_ID:0:10}"                    
+      -
+        name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          registry: artifacts.iflytek.com
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Build and push
+        run: |
+          docker version
+          docker buildx build -t artifacts.iflytek.com/docker-private/atp/${{ steps.image-info.outputs.image_name }}:${{ steps.image-info.outputs.image_tag }} . --file ${{ github.workspace }}/Dockerfile --load
+          docker push artifacts.iflytek.com/docker-private/atp/${{ steps.image-info.outputs.image_name }}:${{ steps.image-info.outputs.image_tag }}
+          docker rmi artifacts.iflytek.com/docker-private/atp/${{ steps.image-info.outputs.image_name }}:${{ steps.image-info.outputs.image_tag }}                    
+      - run: echo "🍏 This job's status is ${{ job.status }}."

Dockerfile

@@ -0,0 +1,11 @@
+#FROM python:3.8.13
+FROM artifacts.iflytek.com/docker-private/atp/base_image_for_ailab:0.0.1
+
+WORKDIR /app
+
+COPY . /app
+
+RUN pip config set global.index-url https://pypi.mirrors.ustc.edu.cn/simple 
+RUN pip install -r requirements.txt
+
+CMD ["python", "app.py"]

api/README.md

@@ -0,0 +1,38 @@
+## 运行环境
+python3.8
+## 项目目录介绍
+### requirements.txt
+项目依赖包管理文件<br>
+首次下载此项目后执行`pip3 install -r requirements.txt`即下载安装依赖包
+
+### data.py
+根据接口文档自动生成的相关配置（请根据接口文档配置所需的请求参数）
+- 请求协议
+- 请求地址
+- 响应数据段，便于程序处理接口响应
+- APPId、APIKey、APISecret信息
+
+### main.py
+程序的执行入口
+
+### resource目录
+此目录提供平台内置的请求文件以及后续保存程序执行响应文件
+
+### sample目录
+程序核心执行逻辑步骤
+
+#### exception.py
+自定义异常
+
+#### ne_utils.py
+工具文件 
+- 读取问题获取文件内容
+- 清空目录
+- 生成鉴权的url
+- 解析url获取对应的host、path、schema
+
+#### aipaas_client.py
+核心程序执行
+- 数据准备 prepare_req_data
+- 执行 execute
+- 处理响应数据 deal_response

api/data.py

@@ -0,0 +1,44 @@
+APPId = "00b49edc"
+APIKey = "6092a002d53b61aa843a4938f24a7edd"
+APISecret = "NGM1N2M3MTU0Mzg3NjZiNTFjZWZhZDE5"
+
+# 请求数据
+request_data = {
+	"header":{
+		"app_id":"123456",
+		"uid":"39769795890",
+		"did":"SR082321940000200",
+		"imei":"8664020318693660",
+		"imsi":"4600264952729100",
+		"mac":"6c:92:bf:65:c6:14",
+		"net_type":"wifi",
+		"net_isp":"CMCC",
+		"status":3,
+		"res_id":""
+	},
+	"parameter":{
+		"sede7bced":{
+			"res_data":{
+				"encoding":"utf8",
+				"compress":"raw",
+				"format":"json"
+			}
+		}
+	},
+	"payload":{
+		"input_text":{
+			"encoding":"utf8",
+			"compress":"raw",
+			"format":"plain",
+			"status":3,
+			"text":"./resource/input/text/阳光总在风雨后.txt"
+		}
+	}
+}
+
+# 请求地址
+request_url = "https://cn-huadong-1.xf-yun.com/v1/private/sede7bced"
+
+# 用于快速定位响应值
+
+response_path_list = ['$..payload.res_data', ]

api/main.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+# -*-coding:utf-8 -*-
+from sample.aipass_client import execute_str
+from data import *
+
+
+def correct_api(text):
+    request_data['header']['app_id'] = '00b49edc'
+    return execute_str(request_url, request_data, "POST", APPId, APIKey, APISecret, text)
+
+
+if __name__ == '__main__':
+    print(correct_api('我真的很系欢你哦'))

api/requirements.txt

@@ -0,0 +1,2 @@
+jsonpath_rw==1.4.0
+requests==2.26.0

api/resource/input/text/╤Ї╣т╫▄╘┌╖ч╙ъ║є.txt

@@ -0,0 +1 @@
+你好，我是艺名小学生。

api/resource/output/res_data.utf8

@@ -0,0 +1 @@
+{"error_details": [{"index_start": 5, "index_end": 7, "error_type": 0, "error_desc": "字词错误", "source_text": "艺名", "have_target": true, "target_text": ["佚名"]}], "error_count": 1}

api/sample/aipass_client.py

@@ -0,0 +1,153 @@
+import base64
+import copy
+import json
+
+import requests
+import jsonpath_rw
+
+from sample import ne_utils
+from data import response_path_list
+from urllib import parse
+
+# media_type_list = ["text", "audio", "image", "video"]
+media_type_list = ["text"]
+
+
+# 准备请求数据
+def prepare_req_data(request_data):
+    new_request_data = copy.deepcopy(request_data)
+    media_path2name = {}
+    for media_type in media_type_list:
+        media_expr = jsonpath_rw.parse("$..payload.*.{}".format(media_type))
+        media_match = media_expr.find(new_request_data)
+        if len(media_match) > 0:
+            for media in media_match:
+                media_path2name[str(media.full_path)] = media.value
+    for media_path, media_name in media_path2name.items():
+        payload_path_list = media_path.split(".")
+        f_data = ne_utils.get_file_bytes(media_name)
+        new_request_data['header']['status'] = 3
+        new_request_data['payload'][payload_path_list[1]][payload_path_list[2]] = base64.b64encode(f_data).decode()
+        new_request_data['payload'][payload_path_list[1]]['status'] = 3
+    return new_request_data
+
+def prepare_req_str_data(request_data, text):
+    new_request_data = copy.deepcopy(request_data)
+    media_path2name = {}
+    for media_type in media_type_list:
+        media_expr = jsonpath_rw.parse("$..payload.*.{}".format(media_type))
+        media_match = media_expr.find(new_request_data)
+        if len(media_match) > 0:
+            for media in media_match:
+                media_path2name[str(media.full_path)] = media.value
+    for media_path, media_name in media_path2name.items():
+        payload_path_list = media_path.split(".")
+        f_data = bytes(text, 'UTF-8')
+        new_request_data['header']['status'] = 3
+        new_request_data['payload'][payload_path_list[1]][payload_path_list[2]] = base64.b64encode(f_data).decode()
+        new_request_data['payload'][payload_path_list[1]]['status'] = 3
+    return new_request_data
+
+# 执行http请求
+def execute(request_url, request_data, method, app_id, api_key, api_secret):
+    # 清除文件
+    ne_utils.del_file('./resource/output')
+
+    # 获取请求url
+    auth_request_url = ne_utils.build_auth_request_url(request_url, method, api_key, api_secret)
+
+    url_result = parse.urlparse(request_url)
+    headers = {'content-type': "application/json", 'host': url_result.hostname, 'app_id': app_id}
+    # 准备待发送的数据
+    new_request_data = prepare_req_data(request_data)
+    print("请求数据:{}\n".format(new_request_data))
+    response = requests.post(auth_request_url, data=json.dumps(new_request_data), headers=headers)
+    deal_response(response)
+
+def execute_str(request_url, request_data, method, app_id, api_key, api_secret, text):
+    # 获取请求url
+    auth_request_url = ne_utils.build_auth_request_url(request_url, method, api_key, api_secret)
+
+    url_result = parse.urlparse(request_url)
+    headers = {'content-type': "application/json", 'host': url_result.hostname, 'app_id': app_id}
+    # 准备待发送的数据
+    new_request_data = prepare_req_str_data(request_data, text)
+    print("请求数据:{}\n".format(new_request_data))
+    response = requests.post(auth_request_url, data=json.dumps(new_request_data), headers=headers)
+    return deal_str_response(response)
+
+# 处理响应数据
+def deal_response(response):
+    temp_result = json.loads(response.content.decode())
+    print("响应数据:{}\n".format(temp_result))
+    header = temp_result.get('header')
+    if header is None:
+        return
+    code = header.get('code')
+    if header is None or code != 0:
+        print("获取结果失败，请根据code查证问题原因")
+        return
+    print("sid:{}".format(header.get('sid')))
+
+    # 打印Base64解码后数据并生成文件
+    if response_path_list is None or len(response_path_list) == 0:
+        return
+    for response_path in response_path_list:
+        response_expr = jsonpath_rw.parse(response_path)
+        response_match = response_expr.find(temp_result)
+        if len(response_match) > 0:
+            for response_item in response_match:
+                if response_item.value is None:
+                    continue
+                encoding = response_item.value.get('encoding')
+                if encoding is None or len(encoding) == 0:
+                    continue
+                for media_type in media_type_list:
+                    media_value = response_item.value.get(media_type)
+                    if media_value is None or len(media_value) == 0:
+                        continue
+                    real_data = base64.b64decode(media_value)
+                    response_path_split_list = response_path.split('.')
+                    write_file_path = "./resource/output/{}.{}".\
+                        format(response_path_split_list[len(response_path_split_list)-1], encoding)
+                    with open(write_file_path, "ab") as file:
+                        file.write(real_data)
+
+def deal_str_response(response):
+    temp_result = json.loads(response.content.decode())
+    # print("响应数据:{}\n".format(temp_result))
+    header = temp_result.get('header')
+    if header is None:
+        return
+    code = header.get('code')
+    if header is None or code != 0:
+        print("获取结果失败，请根据code查证问题原因")
+        return
+    # print("sid:{}".format(header.get('sid')))
+
+    # 打印Base64解码后数据并生成文件
+    if response_path_list is None or len(response_path_list) == 0:
+        return
+    for response_path in response_path_list:
+        response_expr = jsonpath_rw.parse(response_path)
+        response_match = response_expr.find(temp_result)
+        if len(response_match) > 0:
+            for response_item in response_match:
+                if response_item.value is None:
+                    continue
+                encoding = response_item.value.get('encoding')
+                if encoding is None or len(encoding) == 0:
+                    continue
+                for media_type in media_type_list:
+                    media_value = response_item.value.get(media_type)
+                    if media_value is None or len(media_value) == 0:
+                        continue
+                    real_data = base64.b64decode(media_value)
+                    '''
+                    response_path_split_list = response_path.split('.')
+                    write_file_path = "./resource/output/{}.{}".\
+                        format(response_path_split_list[len(response_path_split_list)-1], encoding)
+                    with open(write_file_path, "ab") as file:
+                        file.write(real_data)
+                    '''
+                    return real_data.decode('UTF-8')

api/sample/exception.py

@@ -0,0 +1,8 @@
+class AssembleHeaderException(Exception):
+    def __init__(self, msg):
+        self.message = msg
+
+
+class FileNotFoundException(Exception):
+    def __init__(self, msg):
+        self.message = msg

api/sample/ne_utils.py

@@ -0,0 +1,74 @@
+import base64
+import hashlib
+import hmac
+import os
+import shutil
+from datetime import datetime
+from time import mktime
+from urllib.parse import urlencode
+from wsgiref.handlers import format_date_time
+
+from sample.exception import FileNotFoundException, AssembleHeaderException
+from urllib import parse
+
+
+def get_file_bytes(fd):
+    """
+    根据文件路径（一般相对）获取二进制数据
+    :param fd: 文件路径
+    :return: 二进制数据，不存在时为 None供后续判断
+    """
+    if os.path.exists(fd):
+        with open(fd, "rb") as f:
+            wav_maker = f.read(4)
+            if b'RIFF' == wav_maker:
+                f.seek(44, 0)
+            else:
+                f.seek(0, 0)
+            f_data = f.read()
+        f.close()
+    else:
+        raise FileNotFoundException("{}：文件不存在".format(fd))
+    return f_data
+
+def get_str_bytes():
+    string = ''
+    print('请输入：',end = '')
+    for s in iter(input, ''):
+        string += (s + '\n')
+    s1 = bytes(string, 'UTF-8')
+    return s1
+
+
+def del_file(filepath):
+    """
+    删除某一目录下的所有文件或文件夹
+    :param filepath: 路径
+    :return:
+    """
+    del_list = os.listdir(filepath)
+    for f in del_list:
+        file_path = os.path.join(filepath, f)
+        if os.path.isfile(file_path):
+            os.remove(file_path)
+        elif os.path.isdir(file_path):
+            shutil.rmtree(file_path)
+
+
+# 生成鉴权的url
+def build_auth_request_url(request_url, method="POST", api_key="", api_secret=""):
+    url_result = parse.urlparse(request_url)
+    date = format_date_time(mktime(datetime.now().timetuple()))
+    signature_origin = "host: {}\ndate: {}\n{} {} HTTP/1.1".format(url_result.hostname, date, method, url_result.path)
+    signature_sha = hmac.new(api_secret.encode('utf-8'), signature_origin.encode('utf-8'),
+                             digestmod=hashlib.sha256).digest()
+    signature_sha = base64.b64encode(signature_sha).decode(encoding='utf-8')
+    authorization_origin = "api_key=\"%s\", algorithm=\"%s\", headers=\"%s\", signature=\"%s\"" % (
+        api_key, "hmac-sha256", "host date request-line", signature_sha)
+    authorization = base64.b64encode(authorization_origin.encode('utf-8')).decode(encoding='utf-8')
+    values = {
+        "host": url_result.hostname,
+        "date": date,
+        "authorization": authorization
+    }
+    return request_url + "?" + urlencode(values)

app.py

@@ -0,0 +1,115 @@
+import gradio as gr
+import json
+import random
+import sys
+sys.path.append('./api')
+from main import correct_api
+
+# def correct(text='今天上学不要赤道啊'):
+def correct(text):
+    error_json = json.loads(correct_api(text))
+    print(error_json)
+    error_json_res = {}
+    error_json_res['error_details'] = []
+    for item in error_json["error_details"]:
+        if item['have_target']:
+            item['target_text'] = item['target_text'][0]
+            error_json_res['error_details'].append(item)
+    error_json_res["error_count"] = len(error_json_res["error_details"])
+    # return {
+    #     "error_count": 1,
+    #     "error_details":[{
+    #             "index_start": 6,
+    #             "index_end": 8,
+    #             "source_text": "赤道",
+    #             "target_text": "迟到"}]
+    # }
+    return error_json_res
+
+def convert_to_ner(text):
+    model_output = correct(text)
+    model_output['text'] = text
+    res_dict = {}
+    res_dict['text'] = text
+    res_dict['entities'] = []
+    for item in model_output['error_details']:
+        insert_item = {}
+        insert_item['entity'] = item['target_text']
+        insert_item['start'] = item['index_start']
+        insert_item['end'] = item['index_end']
+        res_dict['entities'].append(insert_item)
+    return res_dict, model_output, get_error_item(model_output)
+
+def get_error_item(error_json):
+    if error_json['error_details']:
+        item = error_json['error_details'][0]
+        res_dict = {}
+        res_dict['text'] = f'({1+error_json["error_count"]-len(error_json["error_details"])}/{error_json["error_count"]})建议将{item["source_text"]}修改为{item["target_text"]}'
+        res_dict['entities'] = []
+        prefix = len(f'({1+error_json["error_count"]-len(error_json["error_details"])}/{error_json["error_count"]})建议将')
+        insert_item = {}
+        insert_item['entity'] = 'wrong'
+        insert_item['start'] = prefix
+        insert_item['end'] = prefix + len(item["source_text"])
+        res_dict['entities'].append(insert_item)
+        prefix += 3
+        insert_item = {}
+        insert_item['entity'] = 'right'
+        insert_item['start'] = prefix + len(item["source_text"])
+        insert_item['end'] = prefix + len(item["source_text"]) + len(item["target_text"])
+        res_dict['entities'].append(insert_item)
+        return res_dict
+    else:
+        return {'text':'未发现错误：）', 'entities':[{'entity':'right', 'start':0, 'end':7}]}
+
+def get_tri_res(error_json):
+    res_dict = {}
+    res_dict['text'] = error_json['text']
+    res_dict['entities'] = []
+    for item in error_json['error_details']:
+        insert_item = {}
+        insert_item['entity'] = item['target_text']
+        insert_item['start'] = item['index_start']
+        insert_item['end'] = item['index_end']
+        res_dict['entities'].append(insert_item)
+    return res_dict, error_json, get_error_item(error_json)
+
+def accept(error_json):
+    raw_text = error_json['text']
+    if error_json['error_details']:
+        item_card = error_json['error_details'].pop(0)
+        error_json['text'] = raw_text[:item_card['index_start']] + item_card['target_text'] + raw_text[item_card['index_end']:]
+        if item_card['target_text'] != item_card['source_text']:
+            dif = len(item_card['target_text']) - len(item_card['source_text'])
+            for item in error_json['error_details']:
+                item['index_start'] += dif
+                item['index_end'] += dif
+    return get_tri_res(error_json)
+
+def reject(error_json):
+    if error_json['error_details']:
+        _ = error_json['error_details'].pop(0)
+    return get_tri_res(error_json)
+
+if __name__ == '__main__':
+    with gr.Blocks() as demo:
+        with gr.Row():
+            gr.Markdown('# <center> 文本智能校对Demo')
+        with gr.Row():
+            with gr.Column(scale=1):
+                # input_text = gr.Textbox(label='input', lines=25, max_lines=25, placeholder='请输入待校对的文本：').style(show_copy_button=True)
+                input_text = gr.Textbox(label='input', placeholder='请输入待校对的文本：').style(show_copy_button=True)
+                button_submit = gr.Button(value="check")
+            with gr.Column(scale=1):
+                diaplay = gr.HighlightedText(label='result', show_label=True)
+                hidden_text = gr.JSON(visible=False)
+                item_card = gr.HighlightedText(label='error_item',show_label=False).style(color_map={'wrong':'red', 'right':'green'})
+                with gr.Row():
+                    ac_button = gr.Button(value="accept")
+                    rj_button = gr.Button(value="reject")
+        
+        button_submit.click(convert_to_ner, inputs=input_text, outputs=[diaplay, hidden_text, item_card])
+        ac_button.click(accept, inputs=hidden_text, outputs=[diaplay, hidden_text, item_card])
+        rj_button.click(reject, inputs=hidden_text, outputs=[diaplay, hidden_text, item_card])
+                        
+    demo.launch(server_name='0.0.0.0')

requirements.txt

@@ -0,0 +1,2 @@
+jsonpath_rw==1.4.0
+requests==2.26.0