中文问题相似度.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import distance  \n",
    "import Levenshtein\n",
    "import time\n",
    "import lightgbm as lgb\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import xgboost as xgb\n",
    "from catboost import CatBoostClassifier\n",
    "from numba import jit\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: python-Levenshtein in e:\\programdata\\anaconda3\\lib\\site-packages (0.12.0)\n",
      "Requirement already satisfied: textdistance in e:\\programdata\\anaconda3\\lib\\site-packages (4.2.1)\n"
     ]
    }
   ],
   "source": [
    "# !pip install distance\n",
    "!pip install python-Levenshtein textdistance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 加载数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train=pd.read_csv('data/train.csv',sep='\\t',header=None)\n",
    "train.columns=['q1','q2','label']\n",
    "test=pd.read_csv('data/test.csv',sep='\\t',header=None)\n",
    "test.columns=['q1','q2']\n",
    "test['label']=1\n",
    "sample_submit=pd.read_csv('data/sample_submit.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>q1</th>\n",
       "      <th>q2</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>有哪些女明星被潜规则啦</td>\n",
       "      <td>哪些女明星被潜规则了</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>怎么支付宝绑定银行卡？</td>\n",
       "      <td>银行卡怎么绑定支付宝</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>请问这部电视剧叫什么名字</td>\n",
       "      <td>请问谁知道这部电视剧叫什么名字</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>泰囧完整版下载</td>\n",
       "      <td>エウテルペ完整版下载</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>在沧州市区哪家卖的盐焗鸡好吃？</td>\n",
       "      <td>沧州饭店哪家便宜又好吃又实惠</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                q1               q2  label\n",
       "0      有哪些女明星被潜规则啦       哪些女明星被潜规则了      1\n",
       "1      怎么支付宝绑定银行卡？       银行卡怎么绑定支付宝      1\n",
       "2     请问这部电视剧叫什么名字  请问谁知道这部电视剧叫什么名字      1\n",
       "3          泰囧完整版下载       エウテルペ完整版下载      0\n",
       "4  在沧州市区哪家卖的盐焗鸡好吃？   沧州饭店哪家便宜又好吃又实惠      0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 5000 entries, 0 to 4999\n",
      "Data columns (total 3 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   q1      5000 non-null   object\n",
      " 1   q2      5000 non-null   object\n",
      " 2   label   5000 non-null   int64 \n",
      "dtypes: int64(1), object(2)\n",
      "memory usage: 117.3+ KB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 5000 entries, 0 to 4999\n",
      "Data columns (total 3 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   q1      5000 non-null   object\n",
      " 1   q2      5000 non-null   object\n",
      " 2   label   5000 non-null   int64 \n",
      "dtypes: int64(1), object(2)\n",
      "memory usage: 117.3+ KB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    0.5784\n",
       "0    0.4216\n",
       "Name: label, dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['label'].value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.concat([train,test],axis=0).reset_index(drop=True)\n",
    "train_size=len(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 特征工程"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1 基础特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 文本长度特征\n",
    "data['q1_len']=data['q1'].astype(str).map(len)\n",
    "data['q2_len']=data['q2'].astype(str).map(len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    10000.000000\n",
       "mean        10.658400\n",
       "std          4.019095\n",
       "min          3.000000\n",
       "25%          8.000000\n",
       "50%         10.000000\n",
       "75%         12.000000\n",
       "max         49.000000\n",
       "Name: q1_len, dtype: float64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['q1_len'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 长度差特征：差/比例\n",
    "data['q1q2_len_diff']=data['q1_len']-data['q2_len']\n",
    "data['q1q2_len_diff_abs']=np.abs(data['q1_len']-data['q2_len'])\n",
    "data['q1q2_rate']=data['q1_len']/data['q2_len']\n",
    "data['q2q1_rate']=data['q2_len']/data['q1_len']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 特殊符号特征\n",
    "data['q1_end_special']=data['q1'].str.endswith('？').astype(int)\n",
    "data['q2_end_special']=data['q2'].str.endswith('？').astype(int)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2 共现字特征\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['comm_q1q2char_nums']=data.apply(lambda  row:len(set(row['q1'])&set(row['q2'])),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 共现字位置\n",
    "def char_match_pos(q1, q2, pos_i):\n",
    "    q1 = list(q1)\n",
    "    q2 = list(q2)\n",
    "\n",
    "    if pos_i < len(q1):\n",
    "        q2_len = min(len(q2), 25)  # q2_len只匹配前25个字\n",
    "        for pos_j in range(q2_len):\n",
    "            if q1[pos_i] == q2[pos_j]:\n",
    "                q_pos = pos_j + 1  # 如果匹配上了 记录匹配的位置\n",
    "                break\n",
    "            elif pos_j == q2_len - 1:\n",
    "                q_pos = 0  # 如果没有匹配上 赋值为0\n",
    "    else:\n",
    "        q_pos = -1  # 如果后续长度不存在 赋值为-1\n",
    "\n",
    "    return q_pos\n",
    "\n",
    "\n",
    "for pos_i in range(8):\n",
    "    data['q1_pos_' + str(pos_i + 1)] = data.apply(\n",
    "        lambda row: char_match_pos(row['q1'], row['q2'], pos_i), axis=1).astype(np.int8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 这里也可以用结巴分词，改成“词”粒度的"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3 距离特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "距离特征:   0%|                                                                                                                                                                                                                                             | 0/4 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===========距离特征 =============\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "距离特征: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:07<00:00,  1.80s/it]\n"
     ]
    }
   ],
   "source": [
    "print(\"===========距离特征 =============\")\n",
    "sim_func_dict = {\"jaccard\": distance.jaccard,\n",
    "                 \"sorensen\": distance.sorensen,\n",
    "                 \"levenshtein\": distance.levenshtein,\n",
    "                 \"ratio\": Levenshtein.ratio\n",
    "                 }\n",
    "\n",
    "for sim_func in tqdm(sim_func_dict, desc=\"距离特征\"):\n",
    "    data[sim_func] = data.apply(lambda row: sim_func_dict[sim_func](row[\"q1\"],row[\"q2\"]), axis=1)\n",
    "    qt = [[3, 3], [3, 5], [5, 5], [5, 10], [10, 10], [10, 15], [15, 15], [15, 25]]\n",
    "\n",
    "    for qt_len in qt:\n",
    "        if qt_len[0] == 3 and sim_func == \"levenshtein\":\n",
    "            pass\n",
    "        else:\n",
    "            data[sim_func + '_q' + str(qt_len[0]) + '_t' + str(qt_len[1])] = data.apply(\n",
    "                lambda row: sim_func_dict[sim_func](row[\"q1\"][:qt_len[0]],\n",
    "                                                    row[\"q2\"][:qt_len[1]]),\n",
    "                axis=1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4 文本向量匹配特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "W2V_SIZE=100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import gensim\n",
    "import jieba\n",
    "import numpy as np\n",
    "from gensim.models import KeyedVectors\n",
    "from gensim.models import word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\yanqiang\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.572 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    }
   ],
   "source": [
    "data['q1_words_list']=data['q1'].apply(lambda x:[w for w in jieba.cut(x) if w])\n",
    "data['q2_words_list']=data['q2'].apply(lambda x:[w for w in jieba.cut(x) if w])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20000"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sentences=[]\n",
    "# for sent in (data['q1']+data['q2']):\n",
    "#     sentences.append([w for w in jieba.cut(sent) if w])\n",
    "sentences=data['q1_words_list'].values.tolist()+data['q2_words_list'].values.tolist()\n",
    "len(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists('models'):\n",
    "    os.mkdir('models')\n",
    "w2v_model = word2vec.Word2Vec(sentences,\n",
    "                                  size=W2V_SIZE, window=10, min_count=1, workers=4,iter=10, \n",
    "                                  sg=1)\n",
    "w2v_model.save('models/' + 'word2vec.model')\n",
    "w2v_model.wv.save_word2vec_format('models/' + 'word2vec.txt', binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11027"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(w2v_model.wv.index2word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\ProgramData\\Anaconda3\\lib\\site-packages\\tqdm\\std.py:670: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
      "  from pandas import Panel\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:01<00:00, 9291.29it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:01<00:00, 9221.30it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 11409.65it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 11472.44it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 11517.76it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 11752.66it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:01<00:00, 8237.26it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 11552.15it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:01<00:00, 9731.20it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 15511.74it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 15940.08it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 12577.18it/s]\n"
     ]
    }
   ],
   "source": [
    "from scipy.spatial.distance import cosine, cityblock, canberra, euclidean, \\\n",
    "    minkowski, braycurtis, correlation, chebyshev, jensenshannon, mahalanobis, \\\n",
    "    seuclidean, sqeuclidean\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "tqdm.pandas()\n",
    "\n",
    "# 计算词向量的相似度\n",
    "def get_w2v(query, title, num):\n",
    "    q = np.zeros(W2V_SIZE)\n",
    "    count = 0\n",
    "    for w in query:\n",
    "        if w in w2v_model.wv:\n",
    "            q += w2v_model.wv[w]\n",
    "            count += 1\n",
    "    if count == 0:\n",
    "        query_vec = q\n",
    "    query_vec = (q / count).tolist()\n",
    "\n",
    "    t = np.zeros(W2V_SIZE)\n",
    "    count = 0\n",
    "    for w in title:\n",
    "        if w in w2v_model.wv:\n",
    "            t += w2v_model.wv[w]\n",
    "            count += 1\n",
    "    if count == 0:\n",
    "        title_vec = q\n",
    "    title_vec = (t / count).tolist()\n",
    "\n",
    "    if num == 1:\n",
    "        try:\n",
    "            vec_cosine = cosine(query_vec, title_vec)\n",
    "            return vec_cosine\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 2:\n",
    "        try:\n",
    "            vec_canberra = canberra(query_vec, title_vec) / len(query_vec)\n",
    "            return vec_canberra\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 3:\n",
    "        try:\n",
    "            vec_cityblock = cityblock(query_vec, title_vec) / len(query_vec)\n",
    "            return vec_cityblock\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 4:\n",
    "        try:\n",
    "            vec_euclidean = euclidean(query_vec, title_vec)\n",
    "            return vec_euclidean\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 5:\n",
    "        try:\n",
    "            vec_braycurtis = braycurtis(query_vec, title_vec)\n",
    "            return vec_braycurtis\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 6:\n",
    "        try:\n",
    "            vec_minkowski = minkowski(query_vec, title_vec)\n",
    "            return vec_minkowski\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 7:\n",
    "        try:\n",
    "            vec_correlation = correlation(query_vec, title_vec)\n",
    "            return vec_correlation\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "\n",
    "    if num == 8:\n",
    "        try:\n",
    "            vec_chebyshev = chebyshev(query_vec, title_vec)\n",
    "            return vec_chebyshev\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "\n",
    "    if num == 9:\n",
    "        try:\n",
    "            vec_jensenshannon = jensenshannon(query_vec, title_vec)\n",
    "            return vec_jensenshannon\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "\n",
    "    if num == 10:\n",
    "        try:\n",
    "            vec_mahalanobis = mahalanobis(query_vec, title_vec)\n",
    "            return vec_mahalanobis\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "\n",
    "    if num == 11:\n",
    "        try:\n",
    "            vec_seuclidean = seuclidean(query_vec, title_vec)\n",
    "            return vec_seuclidean\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 12:\n",
    "        try:\n",
    "            vec_sqeuclidean = sqeuclidean(query_vec, title_vec)\n",
    "            return vec_sqeuclidean\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "# 词向量的相似度特征\n",
    "data['vec_cosine'] = data.progress_apply(lambda index: get_w2v(index['q1_words_list'], index['q2_words_list'], 1),\n",
    "                                         axis=1)\n",
    "data['vec_canberra'] = data.progress_apply(\n",
    "    lambda index: get_w2v(index['q1_words_list'], index['q2_words_list'], 2), axis=1)\n",
    "data['vec_cityblock'] = data.progress_apply(\n",
    "    lambda index: get_w2v(index['q1_words_list'], index['q2_words_list'], 3), axis=1)\n",
    "data['vec_euclidean'] = data.progress_apply(\n",
    "    lambda index: get_w2v(index['q1_words_list'], index['q2_words_list'], 4), axis=1)\n",
    "data['vec_braycurtis'] = data.progress_apply(\n",
    "    lambda index: get_w2v(index['q1_words_list'], index['q2_words_list'], 5), axis=1)\n",
    "data['vec_minkowski'] = data.progress_apply(\n",
    "    lambda index: get_w2v(index['q1_words_list'], index['q2_words_list'], 6), axis=1)\n",
    "data['vec_correlation'] = data.progress_apply(\n",
    "    lambda index: get_w2v(index['q1_words_list'], index['q2_words_list'], 7), axis=1)\n",
    "\n",
    "data['vec_chebyshev'] = data.progress_apply(\n",
    "    lambda index: get_w2v(index['q1_words_list'], index['q2_words_list'], 8), axis=1)\n",
    "data['vec_jensenshannon'] = data.progress_apply(\n",
    "    lambda index: get_w2v(index['q1_words_list'], index['q2_words_list'], 9), axis=1)\n",
    "data['vec_mahalanobis'] = data.progress_apply(\n",
    "    lambda index: get_w2v(index['q1_words_list'], index['q2_words_list'], 10), axis=1)\n",
    "data['vec_seuclidean'] = data.progress_apply(\n",
    "    lambda index: get_w2v(index['q1_words_list'], index['q2_words_list'], 11), axis=1)\n",
    "data['vec_sqeuclidean'] = data.progress_apply(\n",
    "    lambda index: get_w2v(index['q1_words_list'], index['q2_words_list'], 12), axis=1)\n",
    "\n",
    "data['vec_cosine'] = data['vec_cosine'].astype('float32')\n",
    "data['vec_canberra'] = data['vec_canberra'].astype('float32')\n",
    "data['vec_cityblock'] = data['vec_cityblock'].astype('float32')\n",
    "data['vec_euclidean'] = data['vec_euclidean'].astype('float32')\n",
    "data['vec_braycurtis'] = data['vec_braycurtis'].astype('float32')\n",
    "data['vec_correlation'] = data['vec_correlation'].astype('float32')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5 向量特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 10810.74it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 10891.13it/s]\n"
     ]
    }
   ],
   "source": [
    "def w2v_sent2vec(words):\n",
    "    \"\"\"计算句子的平均word2vec向量, sentences是一个句子, 句向量最后会归一化\"\"\"\n",
    "\n",
    "    M = []\n",
    "    for word in words:\n",
    "        try:\n",
    "            M.append(w2v_model.wv[word])\n",
    "        except KeyError:  # 不在词典里\n",
    "            continue\n",
    "\n",
    "    M = np.array(M)\n",
    "    v = M.sum(axis=0)\n",
    "    return (v / np.sqrt((v ** 2).sum())).astype(np.float32).tolist()\n",
    "\n",
    "\n",
    "fea_names = ['q1_vec_{}'.format(i) for i in range(W2V_SIZE)]\n",
    "data[fea_names] = data.progress_apply(lambda row: w2v_sent2vec(row['q1_words_list']), result_type='expand', axis=1)\n",
    "\n",
    "fea_names = ['q2_vec_{}'.format(i) for i in range(W2V_SIZE)]\n",
    "data[fea_names] = data.progress_apply(lambda row: w2v_sent2vec(row['q2_words_list']), result_type='expand', axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "#### tfidf 向量特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "n_componets=16\n",
    "\n",
    "def train_tfidf_model():\n",
    "    tfidf_vectorizer=TfidfVectorizer(min_df=5,ngram_range=(1,2),max_features=100000)\n",
    "    tfidf_array=tfidf_vectorizer.fit_transform([\" \".join(sent) for sent in sentences])\n",
    "    svd = TruncatedSVD(n_components=n_componets, n_iter=7, random_state=42)\n",
    "    tfidf_array=svd.fit_transform(tfidf_array)\n",
    "    return tfidf_vectorizer,svd\n",
    "tfidf_vectorizer,svd=train_tfidf_model()\n",
    "\n",
    "# data['q1_words_list'].values.tolist()+data['q2_words_list'].values.tolist()\n",
    "q1_tfidf_array=tfidf_vectorizer.transform([\" \".join(sent) for sent in data['q1_words_list']])\n",
    "q1_tfidf_array=svd.transform(q1_tfidf_array)\n",
    "# q1_tfidf_df=pd.DataFrame(q1_tfidf_array,columns=[f'q1_tfidf_vec_{i}' for i in range(n_componets)])\n",
    "\n",
    "q2_tfidf_array=tfidf_vectorizer.transform([\" \".join(sent) for sent in data['q2_words_list']])\n",
    "q2_tfidf_array=svd.transform(q2_tfidf_array)\n",
    "# q2_tfidf_df=pd.DataFrame(q2_tfidf_array,columns=[f'q2_tfidf_vec_{i}' for i in range(n_componets)])\n",
    "# data=pd.concat([data,q1_tfidf_df,q2_tfidf_df],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tfidf_sim(row,num):\n",
    "#     print(row.name)\n",
    "    query_vec = q1_tfidf_array[row.name]\n",
    "    title_vec = q2_tfidf_array[row.name]\n",
    "\n",
    "    if num == 1:\n",
    "        try:\n",
    "            vec_cosine = cosine(query_vec, title_vec)\n",
    "            return vec_cosine\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 2:\n",
    "        try:\n",
    "            vec_canberra = canberra(query_vec, title_vec) / len(query_vec)\n",
    "            return vec_canberra\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 3:\n",
    "        try:\n",
    "            vec_cityblock = cityblock(query_vec, title_vec) / len(query_vec)\n",
    "            return vec_cityblock\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 4:\n",
    "        try:\n",
    "            vec_euclidean = euclidean(query_vec, title_vec)\n",
    "            return vec_euclidean\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 5:\n",
    "        try:\n",
    "            vec_braycurtis = braycurtis(query_vec, title_vec)\n",
    "            return vec_braycurtis\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 6:\n",
    "        try:\n",
    "            vec_minkowski = minkowski(query_vec, title_vec)\n",
    "            return vec_minkowski\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 7:\n",
    "        try:\n",
    "            vec_correlation = correlation(query_vec, title_vec)\n",
    "            return vec_correlation\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "\n",
    "    if num == 8:\n",
    "        try:\n",
    "            vec_chebyshev = chebyshev(query_vec, title_vec)\n",
    "            return vec_chebyshev\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "\n",
    "    if num == 9:\n",
    "        try:\n",
    "            vec_jensenshannon = jensenshannon(query_vec, title_vec)\n",
    "            return vec_jensenshannon\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "\n",
    "    if num == 10:\n",
    "        try:\n",
    "            vec_mahalanobis = mahalanobis(query_vec, title_vec)\n",
    "            return vec_mahalanobis\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "\n",
    "    if num == 11:\n",
    "        try:\n",
    "            vec_seuclidean = seuclidean(query_vec, title_vec)\n",
    "            return vec_seuclidean\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 12:\n",
    "        try:\n",
    "            vec_sqeuclidean = sqeuclidean(query_vec, title_vec)\n",
    "            return vec_sqeuclidean\n",
    "        except Exception as e:\n",
    "            return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\ProgramData\\Anaconda3\\lib\\site-packages\\scipy\\spatial\\distance.py:728: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dist = 1.0 - uv / np.sqrt(uu * vv)\n",
      "E:\\ProgramData\\Anaconda3\\lib\\site-packages\\scipy\\spatial\\distance.py:1186: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  return l1_diff.sum() / l1_sum.sum()\n",
      "E:\\ProgramData\\Anaconda3\\lib\\site-packages\\scipy\\spatial\\distance.py:1293: RuntimeWarning: invalid value encountered in true_divide\n",
      "  p = p / np.sum(p, axis=0)\n",
      "E:\\ProgramData\\Anaconda3\\lib\\site-packages\\scipy\\spatial\\distance.py:1294: RuntimeWarning: invalid value encountered in true_divide\n",
      "  q = q / np.sum(q, axis=0)\n"
     ]
    }
   ],
   "source": [
    "data['tfidf_vec_cosine'] = data.apply(lambda index: get_tfidf_sim(index, 1),axis=1)\n",
    "data['tfidf_vec_canberra'] = data.apply(lambda index: get_tfidf_sim(index, 2),axis=1)\n",
    "data['tfidf_vec_cityblock'] = data.apply(lambda index: get_tfidf_sim(index, 3),axis=1)\n",
    "data['tfidf_vec_euclidean'] = data.apply(lambda index: get_tfidf_sim(index, 4),axis=1)\n",
    "data['tfidf_vec_braycurtis'] = data.apply(lambda index: get_tfidf_sim(index, 5),axis=1)\n",
    "data['tfidf_vec_minkowski'] = data.apply(lambda index: get_tfidf_sim(index, 6),axis=1)\n",
    "data['tfidf_vec_correlation'] = data.apply(lambda index: get_tfidf_sim(index, 7),axis=1)\n",
    "\n",
    "data['tfidf_vec_chebyshev'] = data.apply(lambda index: get_tfidf_sim(index, 8),axis=1)\n",
    "data['tfidf_sb_vec_jensenshannon'] = data.apply(lambda index: get_tfidf_sim(index, 9),axis=1)\n",
    "data['tfidf_vec_mahalanobis'] = data.apply(lambda index: get_tfidf_sim(index, 10),axis=1)\n",
    "data['tfidf_vec_seuclidean'] = data.apply(lambda index: get_tfidf_sim(index, 11),axis=1)\n",
    "data['tfidf_vec_sqeuclidean'] = data.apply(lambda index: get_tfidf_sim(index, 12),axis=1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6 词粒度特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['query']=data['q1_words_list'].apply(lambda x:\" \".join(x))\n",
    "data['title']=data['q2_words_list'].apply(lambda x:\" \".join(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 14688.68it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 3302341.55it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 2505557.95it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 19325/19325 [00:00<00:00, 460128.89it/s]\n",
      "  8%|█████████████████▌                                                                                                                                                                                                                     | 759/10000 [00:00<00:01, 7533.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "一共有19325个unique文档.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 13225.72it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 12243.74it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:01<00:00, 8330.86it/s]\n",
      "  1%|███▏                                                                                                                                                                                                                                   | 137/10000 [00:00<00:07, 1361.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:02<00:00, 4442.67it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 13286.06it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10000/10000 [00:00<00:00, 12480.83it/s]\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "import math\n",
    "import time\n",
    "import gc\n",
    "import os\n",
    "import csv\n",
    "import json\n",
    "from itertools import chain\n",
    "from tqdm import tqdm\n",
    "def get_wordshare(row):\n",
    "    query = row['query'].strip().split()\n",
    "    title = row['title'].strip().split()\n",
    "\n",
    "    query_len, title_len = len(query), len(title)\n",
    "\n",
    "    query_words = {}\n",
    "    title_words = {}\n",
    "    for word in query:  # query\n",
    "        query_words[word] = query_words.get(word, 0) + 1\n",
    "    for word in title:  # title\n",
    "        title_words[word] = title_words.get(word, 0) + 1\n",
    "    share_term = set(query_words.keys()) & set(title_words.keys())\n",
    "\n",
    "    # -------------------- WordMatchShare --------------\n",
    "    n_shared_word_in_query = sum([query_words[w] for w in share_term])\n",
    "    n_shared_word_in_title = sum([title_words[w] for w in share_term])\n",
    "\n",
    "    WordMatchShare = (n_shared_word_in_query + n_shared_word_in_title) / (\n",
    "            query_len + title_len)\n",
    "    WordMatchShare_query = n_shared_word_in_query / query_len\n",
    "    WordMatchShare_title = n_shared_word_in_title / title_len\n",
    "    return query_len, title_len, WordMatchShare, WordMatchShare_query, WordMatchShare_title\n",
    "\n",
    "\n",
    "fea_names = ['query_length', 'title_length', 'WordMatchShare', 'WordMatchShare_query',\n",
    "             'WordMatchShare_title'  # 这四个根据前面的计算得到\n",
    "             ]\n",
    "data[fea_names] = data.progress_apply(lambda row: get_wordshare(row), result_type='expand', axis=1)\n",
    "\n",
    "\n",
    "# 长度差特征\n",
    "def get_lendiff(all_data):\n",
    "    all_data['LengthDiff'] = all_data['query_length'] - all_data['title_length']\n",
    "    all_data['LengthDiffRate'] = np.amin(all_data[['query_length', 'title_length']].values, axis=1) / np.amax(\n",
    "        all_data[['query_length', 'title_length']].values, axis=1)\n",
    "\n",
    "    all_data['LengthRatio_qt'] = all_data['query_length'] / all_data['title_length']\n",
    "    all_data['LengthRatio_tq'] = all_data['title_length'] / all_data['query_length']\n",
    "    return all_data\n",
    "\n",
    "\n",
    "data = get_lendiff(data)\n",
    "\n",
    "\n",
    "# tfidf\n",
    "\n",
    "\n",
    "def get_tfidfwordshare(row, idf):\n",
    "    query = row['query'].strip().split()\n",
    "    title = row['title'].strip().split()\n",
    "\n",
    "    query_words = {}\n",
    "    title_words = {}\n",
    "    for word in query:  # query\n",
    "        query_words[word] = query_words.get(word, 0) + 1\n",
    "    for word in title:  # title\n",
    "        title_words[word] = title_words.get(word, 0) + 1\n",
    "    share_term = set(query_words.keys()) & set(title_words.keys())\n",
    "\n",
    "    # -------------------- WordMatchShare --------------\n",
    "    sum_shared_word_in_query = sum([query_words[w] * idf.get(w, 0) for w in share_term])\n",
    "    sum_shared_word_in_title = sum([title_words[w] * idf.get(w, 0) for w in share_term])\n",
    "    sum_query_tol = sum(query_words[w] * idf.get(w, 0) for w in query_words)\n",
    "    sum_title_tol = sum(title_words[w] * idf.get(w, 0) for w in title_words)\n",
    "    sum_tol = sum_query_tol + sum_title_tol\n",
    "\n",
    "    TFIDFWordMatchShare = (sum_shared_word_in_query + sum_shared_word_in_title) / sum_tol\n",
    "    TFIDFWordMatchShare_query = sum_shared_word_in_query / sum_query_tol\n",
    "    TFIDFWordMatchShare_title = sum_shared_word_in_title / sum_title_tol\n",
    "    return TFIDFWordMatchShare, TFIDFWordMatchShare_query, TFIDFWordMatchShare_title\n",
    "\n",
    "\n",
    "def get_tfidffea(data):\n",
    "    doc_set = set()\n",
    "    for title in tqdm(data['title']):\n",
    "        doc_set.add(title)\n",
    "    for query in tqdm(data['query']):\n",
    "        doc_set.add(query)\n",
    "\n",
    "    idf = {}\n",
    "    doc_len = len(doc_set)\n",
    "    print(\"一共有%d个unique文档.\" % doc_len)\n",
    "    for doc in tqdm(doc_set):\n",
    "        for word in set(doc.split()):\n",
    "            idf[word] = idf.get(word, 0) + 1\n",
    "\n",
    "    for word in idf:\n",
    "        idf[word] = np.log(doc_len / (idf[word] + 1.)) + 1\n",
    "\n",
    "    fea_names = ['TFIDFWordMatchShare', 'TFIDFWordMatchShare_query', 'TFIDFWordMatchShare_title']\n",
    "\n",
    "    data[fea_names] = data.progress_apply(lambda row: get_tfidfwordshare(row, idf), result_type='expand', axis=1)\n",
    "    return data\n",
    "\n",
    "\n",
    "data = get_tfidffea(data)\n",
    "\n",
    "## NgramJaccard特征\n",
    "\n",
    "from nltk.util import bigrams\n",
    "from nltk.util import ngrams\n",
    "from nltk.util import everygrams\n",
    "\n",
    "\n",
    "def retrieve_ngrams(txt, n):\n",
    "    return [txt[i:i + n] for i in range(len(txt) - (n - 1))]\n",
    "\n",
    "\n",
    "def get_ngrams(wordlist, n):\n",
    "    ngrams = []\n",
    "    for i in range(len(wordlist) - (n - 1)):\n",
    "        ngrams.append(wordlist[i:i + n])\n",
    "    return ngrams\n",
    "\n",
    "\n",
    "import textdistance\n",
    "\n",
    "fea_names = ['NgramJaccardCoef_1', 'NgramJaccardCoef_2', 'NgramJaccardCoef_3', 'NgramJaccardCoef_4']\n",
    "\n",
    "\n",
    "def jaccard_coef(aa, bb):\n",
    "    countUp = 0\n",
    "    countDown = 0\n",
    "\n",
    "    # Calculate how many elements in ListB are not in ListA\n",
    "    for i in range(0, len(aa)):\n",
    "        if aa[i] not in bb:\n",
    "            countDown = countDown + 1\n",
    "\n",
    "    # Calculate the numbers of common element\n",
    "    for i in range(0, len(bb)):\n",
    "        if bb[i] not in aa:\n",
    "            countUp = countUp + 1\n",
    "\n",
    "    # String_1 intersect String_2\n",
    "    Numerator = len(aa) - countDown\n",
    "\n",
    "    # String_1 union String_2\n",
    "    Denominator = len(aa) + countUp\n",
    "\n",
    "    jaccard = Numerator * 1.0 / (Denominator + 1)\n",
    "    return jaccard\n",
    "\n",
    "\n",
    "def get_ngramjaccard(row):\n",
    "    query = row['query'].strip().split()\n",
    "    title = row['title'].strip().split()\n",
    "\n",
    "    query_ngrams = get_ngrams(query, 1)\n",
    "    title_ngrams = get_ngrams(title, 1)\n",
    "    NgramJaccardCoef_1 = jaccard_coef(query_ngrams,\n",
    "                                      title_ngrams)\n",
    "    query_ngrams = get_ngrams(query, 2)\n",
    "    title_ngrams = get_ngrams(title, 2)\n",
    "    NgramJaccardCoef_2 = jaccard_coef(query_ngrams,\n",
    "                                      title_ngrams)\n",
    "\n",
    "    query_ngrams = get_ngrams(query, 3)\n",
    "    title_ngrams = get_ngrams(title, 3)\n",
    "    NgramJaccardCoef_3 = jaccard_coef(query_ngrams,\n",
    "                                      title_ngrams)\n",
    "    query_ngrams = get_ngrams(query, 4)\n",
    "    title_ngrams = get_ngrams(title, 4)\n",
    "    NgramJaccardCoef_4 = jaccard_coef(query_ngrams,\n",
    "                                      title_ngrams)\n",
    "    return NgramJaccardCoef_1, NgramJaccardCoef_2, NgramJaccardCoef_3, NgramJaccardCoef_4\n",
    "\n",
    "\n",
    "data[fea_names] = data.progress_apply(lambda row: get_ngramjaccard(row), result_type='expand', axis=1)\n",
    "\n",
    "\n",
    "# 1.4 sequencematch相关\n",
    "def lcsubstr_lens(s1, s2):  # 计算最长子串长度\n",
    "    m = [[0 for i in range(len(s2) + 1)]\n",
    "         for j in range(len(s1) + 1)]  # 生成0矩阵，为方便后续计算，比字符串长度多了一列\n",
    "    mmax = 0  # 最长匹配的长度\n",
    "    p = 0  # 最长匹配对应在s1中的最后一位\n",
    "    for i in range(len(s1)):\n",
    "        for j in range(len(s2)):\n",
    "            if s1[i] == s2[j]:\n",
    "                m[i + 1][j + 1] = m[i][j] + 1\n",
    "                if m[i + 1][j + 1] > mmax:\n",
    "                    mmax = m[i + 1][j + 1]\n",
    "                    p = i + 1\n",
    "    return mmax\n",
    "\n",
    "\n",
    "def lcseque_lens(s1, s2):  # 计算最长子序列长度\n",
    "    # 生成字符串长度加1的0矩阵，m用来保存对应位置匹配的结果\n",
    "    m = [[0 for x in range(len(s2) + 1)] for y in range(len(s1) + 1)]\n",
    "    # d用来记录转移方向\n",
    "    d = [[None for x in range(len(s2) + 1)] for y in range(len(s1) + 1)]\n",
    "    for p1 in range(len(s1)):\n",
    "        for p2 in range(len(s2)):\n",
    "            if s1[p1] == s2[p2]:  # 字符匹配成功，则该位置的值为左上方的值加1\n",
    "                m[p1 + 1][p2 + 1] = m[p1][p2] + 1\n",
    "                d[p1 + 1][p2 + 1] = 'ok'\n",
    "            elif m[p1 + 1][p2] > m[p1][p2 + 1]:  # 左值大于上值，则该位置的值为左值，并标记回溯时的方向\n",
    "                m[p1 + 1][p2 + 1] = m[p1 + 1][p2]\n",
    "                d[p1 + 1][p2 + 1] = 'left'\n",
    "            else:  # 上值大于左值，则该位置的值为上值，并标记方向up\n",
    "                m[p1 + 1][p2 + 1] = m[p1][p2 + 1]\n",
    "                d[p1 + 1][p2 + 1] = 'up'\n",
    "    (p1, p2) = (len(s1), len(s2))\n",
    "    s = []\n",
    "    while m[p1][p2]:  # 不为None时\n",
    "        c = d[p1][p2]\n",
    "        if c == 'ok':  # 匹配成功，插入该字符，并向左上角找下一个\n",
    "            s.append(s1[p1 - 1])\n",
    "            p1 -= 1\n",
    "            p2 -= 1\n",
    "        if c == 'left':  # 根据标记，向左找下一个\n",
    "            p2 -= 1\n",
    "        if c == 'up':  # 根据标记，向上找下一个\n",
    "            p1 -= 1\n",
    "    return len(s)\n",
    "\n",
    "\n",
    "from difflib import SequenceMatcher\n",
    "\n",
    "\n",
    "def get_sequencematch(row):\n",
    "    query = row['query'].strip().split()\n",
    "    title = row['title'].strip().split()\n",
    "\n",
    "    lcsubstr_len = lcsubstr_lens(query, title)\n",
    "    lcseque_len = lcseque_lens(query, title)\n",
    "    sq = SequenceMatcher(a=query, b=title)\n",
    "    match = sq.find_longest_match(0, len(query), 0, len(title))\n",
    "    longest_match_size = match.size\n",
    "    longest_match_ratio = match.size / min(len(query), len(title))\n",
    "    return lcsubstr_len, lcseque_len, longest_match_size, longest_match_ratio\n",
    "\n",
    "\n",
    "fea_names = [\"lcsubstr_len\", \"lcseque_len\", \"longest_match_size\", \"longest_match_ratio\"]\n",
    "data[fea_names] = data.progress_apply(lambda row: get_sequencematch(row), result_type='expand', axis=1)\n",
    "\n",
    "# Fuzzy特征\n",
    "from fuzzywuzzy import fuzz\n",
    "\n",
    "\n",
    "def get_fuzzz(row):\n",
    "    query_str = row['query'].strip()\n",
    "    query = query_str.split()\n",
    "    title_str = row['title'].strip()\n",
    "    title = title_str.split()\n",
    "\n",
    "    fuzz_qratio = fuzz.QRatio(query_str, title_str)\n",
    "    fuzz_WRatio = fuzz.WRatio(query_str, title_str)\n",
    "    fuzz_partial_ratio = fuzz.partial_ratio(query_str, title_str)\n",
    "\n",
    "    fuzz_partial_token_set_ratio = fuzz.partial_token_set_ratio(query_str, title_str)\n",
    "    fuzz_partial_token_sort_ratio = fuzz.partial_token_sort_ratio(query_str, title_str)\n",
    "    fuzz_token_set_ratio = fuzz.token_set_ratio(query_str, title_str)\n",
    "    fuzz_token_sort_ratio = fuzz.token_sort_ratio(query_str, title_str)\n",
    "\n",
    "    return fuzz_qratio, fuzz_WRatio, fuzz_partial_ratio, fuzz_partial_token_set_ratio, fuzz_partial_token_sort_ratio, \\\n",
    "           fuzz_token_set_ratio, fuzz_token_sort_ratio\n",
    "\n",
    "\n",
    "fea_names = [\"fuzz_qratio\", \"fuzz_WRatio\", \"fuzz_partial_ratio\", \"fuzz_partial_token_set_ratio\",\n",
    "             \"fuzz_partial_token_sort_ratio\",\n",
    "             \"fuzz_token_set_ratio\",\n",
    "             \"fuzz_token_sort_ratio\"]\n",
    "print(len(fea_names))\n",
    "data[fea_names] = data.progress_apply(lambda row: get_fuzzz(row), result_type='expand', axis=1)\n",
    "\n",
    "\n",
    "# 熵相关\n",
    "\n",
    "def get_entropy(row):\n",
    "    query_str = row['query'].strip()\n",
    "    query = query_str.split()\n",
    "    title_str = row['title'].strip()\n",
    "    title = title_str.split()\n",
    "\n",
    "    query_words = {}\n",
    "    title_words = {}\n",
    "    query_title_words = {}\n",
    "    for word in query:  # query\n",
    "        query_words[word] = query_words.get(word, 0) + 1\n",
    "        query_title_words[word] = query_title_words.get(word, 0) + 1\n",
    "    for word in title:  # title\n",
    "        title_words[word] = title_words.get(word, 0) + 1\n",
    "        query_title_words[word] = query_title_words.get(word, 0) + 1\n",
    "\n",
    "    n_query_tol = sum(query_words.values())\n",
    "    n_title_tol = sum(title_words.values())\n",
    "    n_query_title_tol = sum(query_title_words.values())\n",
    "    query_Entropy = abs(sum(map(lambda x: x / n_query_tol * \\\n",
    "                                          math.log(x / n_query_tol, 2),\n",
    "                                query_words.values())))\n",
    "    title_Entropy = abs(sum(map(lambda x: x / n_title_tol * \\\n",
    "                                          math.log(x / n_title_tol, 2),\n",
    "                                title_words.values())))\n",
    "    query_title_Entropy = abs(sum(map(lambda x: x / n_query_title_tol * \\\n",
    "                                                math.log(x / n_query_title_tol, 2),\n",
    "                                      query_title_words.values())))\n",
    "\n",
    "    query_title_words_share = {}\n",
    "    for word in query_words:\n",
    "        if word in title_words:\n",
    "            query_title_words_share[word] = query_title_words_share.get(\n",
    "                word, 0) + query_words[word]\n",
    "    for word in title_words:\n",
    "        if word in query_words:\n",
    "            query_title_words_share[word] = query_title_words_share.get(\n",
    "                word, 0) + title_words[word]\n",
    "\n",
    "    WordMatchShare_Entropy = abs(sum(map(lambda x: x / n_query_title_tol * \\\n",
    "                                                   math.log(x / n_query_title_tol, 2),\n",
    "                                         query_title_words_share.values())))\n",
    "    return query_Entropy, title_Entropy, query_title_Entropy, WordMatchShare_Entropy\n",
    "\n",
    "\n",
    "fea_names = [\"query_Entropy\", \"title_Entropy\", \"query_title_Entropy\", \"WordMatchShare_Entropy\"]\n",
    "data[fea_names] = data.progress_apply(lambda row: get_entropy(row), result_type='expand', axis=1)\n",
    "\n",
    "\n",
    "## 补充特征\n",
    "def get_ngram_rp_query_in_title(query, title):\n",
    "    query = list(query.strip().split())\n",
    "    title = list(title.strip().split())\n",
    "    query_2gram = []\n",
    "    for i in range(len(query) - 1):\n",
    "        query_2gram.append(query[i] + query[i + 1])\n",
    "    query.extend(query_2gram)\n",
    "\n",
    "    title_2gram = []\n",
    "    for i in range(len(title) - 1):\n",
    "        title_2gram.append(title[i] + title[i + 1])\n",
    "    title.extend(title_2gram)\n",
    "\n",
    "    len_query = len(query)\n",
    "    len_title = len(title)\n",
    "    len_common = len(set(query) & set(title))\n",
    "\n",
    "    recall = len_common / (len_query + 0.001)\n",
    "    precision = len_common / (len_title + 0.001)\n",
    "    acc = len_common / (len_query + len_title - len_common)\n",
    "    return [recall, precision, acc]\n",
    "\n",
    "\n",
    "def get_prf(row):\n",
    "    query_set = set(row['query'].strip().split())\n",
    "    title_set = set(row['title'].strip().split())\n",
    "    common_words_len = len(query_set & title_set)\n",
    "    query_len = len(row['query'].strip().split())\n",
    "    title_len = len(row['title'].strip().split())\n",
    "    recall, precision, acc = get_ngram_rp_query_in_title(row['query'], row['title'])\n",
    "\n",
    "    jaccard_similarity = common_words_len / len(query_set | title_set)\n",
    "    qt_coword_query_ratio = common_words_len / query_len\n",
    "    qt_coword_title_ratio = common_words_len / title_len\n",
    "    qt_len_mean = (query_len + title_len) / 2.0\n",
    "    qt_common_word_acc = common_words_len / (query_len + title_len - common_words_len)\n",
    "    ngram_query_title_precision = precision\n",
    "    ngram_query_title_recall = recall\n",
    "    ngram_query_title_acc = acc\n",
    "\n",
    "    return jaccard_similarity, qt_coword_query_ratio, qt_coword_title_ratio, qt_len_mean, \\\n",
    "           qt_common_word_acc, ngram_query_title_precision, ngram_query_title_recall, ngram_query_title_acc\n",
    "\n",
    "\n",
    "\n",
    "fea_names = [\"jaccard_similarity\", \"qt_coword_query_ratio\", \"qt_coword_title_ratio\",\n",
    "             \"qt_len_mean\", \"qt_common_word_acc\",\n",
    "             \"ngram_query_title_precision\", \"ngram_query_title_recall\", \"ngram_query_title_acc\"]\n",
    "data[fea_names] = data.progress_apply(lambda row: get_prf(row), result_type='expand', axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  SentenceTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer, util\n",
    "model = SentenceTransformer('distiluse-base-multilingual-cased-v2')\n",
    "\n",
    "query_embedding = model.encode(data['q1'],show_progress_bar=False)\n",
    "passage_embedding = model.encode(data['q2'],show_progress_bar=False)\n",
    "# q1_sbert_df=pd.DataFrame(query_embedding,columns=[f'q1_sber_vec_{i}' for i in range(512)])\n",
    "# q2_sbert_df=pd.DataFrame(passage_embedding,columns=[f'q2_sber_vec_{i}' for i in range(512)])\n",
    "# data=pd.concat([data,q1_sbert_df,q2_sbert_df],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sbert_sim(row,num):\n",
    "#     print(row.name)\n",
    "    query_vec = query_embedding[row.name]\n",
    "    title_vec = passage_embedding[row.name]\n",
    "\n",
    "    if num == 1:\n",
    "        try:\n",
    "            vec_cosine = cosine(query_vec, title_vec)\n",
    "            return vec_cosine\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 2:\n",
    "        try:\n",
    "            vec_canberra = canberra(query_vec, title_vec) / len(query_vec)\n",
    "            return vec_canberra\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 3:\n",
    "        try:\n",
    "            vec_cityblock = cityblock(query_vec, title_vec) / len(query_vec)\n",
    "            return vec_cityblock\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 4:\n",
    "        try:\n",
    "            vec_euclidean = euclidean(query_vec, title_vec)\n",
    "            return vec_euclidean\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 5:\n",
    "        try:\n",
    "            vec_braycurtis = braycurtis(query_vec, title_vec)\n",
    "            return vec_braycurtis\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 6:\n",
    "        try:\n",
    "            vec_minkowski = minkowski(query_vec, title_vec)\n",
    "            return vec_minkowski\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 7:\n",
    "        try:\n",
    "            vec_correlation = correlation(query_vec, title_vec)\n",
    "            return vec_correlation\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "\n",
    "    if num == 8:\n",
    "        try:\n",
    "            vec_chebyshev = chebyshev(query_vec, title_vec)\n",
    "            return vec_chebyshev\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "\n",
    "    if num == 9:\n",
    "        try:\n",
    "            vec_jensenshannon = jensenshannon(query_vec, title_vec)\n",
    "            return vec_jensenshannon\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "\n",
    "    if num == 10:\n",
    "        try:\n",
    "            vec_mahalanobis = mahalanobis(query_vec, title_vec)\n",
    "            return vec_mahalanobis\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "\n",
    "    if num == 11:\n",
    "        try:\n",
    "            vec_seuclidean = seuclidean(query_vec, title_vec)\n",
    "            return vec_seuclidean\n",
    "        except Exception as e:\n",
    "            return 0\n",
    "    if num == 12:\n",
    "        try:\n",
    "            vec_sqeuclidean = sqeuclidean(query_vec, title_vec)\n",
    "            return vec_sqeuclidean\n",
    "        except Exception as e:\n",
    "            return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['sb_vec_cosine'] = data.apply(lambda index: get_sbert_sim(index, 1),axis=1)\n",
    "data['sb_vec_canberra'] = data.apply(lambda index: get_sbert_sim(index, 2),axis=1)\n",
    "data['sb_sb_sb_vec_cityblock'] = data.apply(lambda index: get_sbert_sim(index, 3),axis=1)\n",
    "data['sb_sb_vec_euclidean'] = data.apply(lambda index: get_sbert_sim(index, 4),axis=1)\n",
    "data['sb_vec_braycurtis'] = data.apply(lambda index: get_sbert_sim(index, 5),axis=1)\n",
    "data['sb_vec_minkowski'] = data.apply(lambda index: get_sbert_sim(index, 6),axis=1)\n",
    "data['sb_vec_correlation'] = data.apply(lambda index: get_sbert_sim(index, 7),axis=1)\n",
    "\n",
    "data['sb_vec_chebyshev'] = data.apply(lambda index: get_sbert_sim(index, 8),axis=1)\n",
    "data['sb_sb_vec_jensenshannon'] = data.apply(lambda index: get_sbert_sim(index, 9),axis=1)\n",
    "data['sb_vec_mahalanobis'] = data.apply(lambda index: get_sbert_sim(index, 10),axis=1)\n",
    "data['sb_vec_seuclidean'] = data.apply(lambda index: get_sbert_sim(index, 11),axis=1)\n",
    "data['sb_vec_sqeuclidean'] = data.apply(lambda index: get_sbert_sim(index, 12),axis=1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### simpletransformers预测结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "torch.cuda.is_available()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.special import softmax\n",
    "tmp_train=[]\n",
    "tmp_test=[]\n",
    "for file in os.listdir('result_kflods_pairs'):\n",
    "#     print(file)\n",
    "    if file.endswith('npy') and 'test' in file:\n",
    "        probs=np.load('result_kflods_pairs/'+file)\n",
    "        probs=softmax(probs,axis=1)[:,1]\n",
    "        tmp_test.append(probs)\n",
    "    if file.endswith('npy') and 'train' in file:\n",
    "        probs=np.load('result_kflods_pairs/'+file)\n",
    "        probs=softmax(probs,axis=1)[:,1]\n",
    "        tmp_train.append(probs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>smbert_prob_0</th>\n",
       "      <th>smbert_prob_1</th>\n",
       "      <th>smbert_prob_2</th>\n",
       "      <th>smbert_prob_3</th>\n",
       "      <th>smbert_prob_4</th>\n",
       "      <th>smbert_prob_5</th>\n",
       "      <th>smbert_prob_6</th>\n",
       "      <th>smbert_prob_7</th>\n",
       "      <th>smbert_prob_8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.721030</td>\n",
       "      <td>0.740094</td>\n",
       "      <td>0.743169</td>\n",
       "      <td>0.705813</td>\n",
       "      <td>0.753971</td>\n",
       "      <td>0.770251</td>\n",
       "      <td>0.768171</td>\n",
       "      <td>0.742659</td>\n",
       "      <td>0.786138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.768305</td>\n",
       "      <td>0.726227</td>\n",
       "      <td>0.718618</td>\n",
       "      <td>0.748502</td>\n",
       "      <td>0.734214</td>\n",
       "      <td>0.757467</td>\n",
       "      <td>0.774990</td>\n",
       "      <td>0.732668</td>\n",
       "      <td>0.727966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.753795</td>\n",
       "      <td>0.747497</td>\n",
       "      <td>0.716905</td>\n",
       "      <td>0.723207</td>\n",
       "      <td>0.715409</td>\n",
       "      <td>0.728038</td>\n",
       "      <td>0.793189</td>\n",
       "      <td>0.733517</td>\n",
       "      <td>0.742522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.428676</td>\n",
       "      <td>0.279509</td>\n",
       "      <td>0.329308</td>\n",
       "      <td>0.381003</td>\n",
       "      <td>0.340693</td>\n",
       "      <td>0.213485</td>\n",
       "      <td>0.429684</td>\n",
       "      <td>0.316582</td>\n",
       "      <td>0.276741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.253364</td>\n",
       "      <td>0.273256</td>\n",
       "      <td>0.243242</td>\n",
       "      <td>0.249783</td>\n",
       "      <td>0.258263</td>\n",
       "      <td>0.234219</td>\n",
       "      <td>0.215183</td>\n",
       "      <td>0.302043</td>\n",
       "      <td>0.201762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>0.113250</td>\n",
       "      <td>0.056813</td>\n",
       "      <td>0.117426</td>\n",
       "      <td>0.200263</td>\n",
       "      <td>0.098060</td>\n",
       "      <td>0.669763</td>\n",
       "      <td>0.037557</td>\n",
       "      <td>0.179920</td>\n",
       "      <td>0.278770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>0.043114</td>\n",
       "      <td>0.011754</td>\n",
       "      <td>0.119420</td>\n",
       "      <td>0.017799</td>\n",
       "      <td>0.163244</td>\n",
       "      <td>0.947966</td>\n",
       "      <td>0.009549</td>\n",
       "      <td>0.160152</td>\n",
       "      <td>0.109565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>0.006674</td>\n",
       "      <td>0.008922</td>\n",
       "      <td>0.007845</td>\n",
       "      <td>0.005696</td>\n",
       "      <td>0.008939</td>\n",
       "      <td>0.132476</td>\n",
       "      <td>0.001972</td>\n",
       "      <td>0.007586</td>\n",
       "      <td>0.003270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>0.002903</td>\n",
       "      <td>0.006609</td>\n",
       "      <td>0.003428</td>\n",
       "      <td>0.005069</td>\n",
       "      <td>0.005045</td>\n",
       "      <td>0.002088</td>\n",
       "      <td>0.000410</td>\n",
       "      <td>0.005311</td>\n",
       "      <td>0.002019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>0.021159</td>\n",
       "      <td>0.009139</td>\n",
       "      <td>0.021746</td>\n",
       "      <td>0.005858</td>\n",
       "      <td>0.013113</td>\n",
       "      <td>0.006877</td>\n",
       "      <td>0.002075</td>\n",
       "      <td>0.018499</td>\n",
       "      <td>0.017215</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      smbert_prob_0  smbert_prob_1  smbert_prob_2  smbert_prob_3  \\\n",
       "0          0.721030       0.740094       0.743169       0.705813   \n",
       "1          0.768305       0.726227       0.718618       0.748502   \n",
       "2          0.753795       0.747497       0.716905       0.723207   \n",
       "3          0.428676       0.279509       0.329308       0.381003   \n",
       "4          0.253364       0.273256       0.243242       0.249783   \n",
       "...             ...            ...            ...            ...   \n",
       "9995       0.113250       0.056813       0.117426       0.200263   \n",
       "9996       0.043114       0.011754       0.119420       0.017799   \n",
       "9997       0.006674       0.008922       0.007845       0.005696   \n",
       "9998       0.002903       0.006609       0.003428       0.005069   \n",
       "9999       0.021159       0.009139       0.021746       0.005858   \n",
       "\n",
       "      smbert_prob_4  smbert_prob_5  smbert_prob_6  smbert_prob_7  \\\n",
       "0          0.753971       0.770251       0.768171       0.742659   \n",
       "1          0.734214       0.757467       0.774990       0.732668   \n",
       "2          0.715409       0.728038       0.793189       0.733517   \n",
       "3          0.340693       0.213485       0.429684       0.316582   \n",
       "4          0.258263       0.234219       0.215183       0.302043   \n",
       "...             ...            ...            ...            ...   \n",
       "9995       0.098060       0.669763       0.037557       0.179920   \n",
       "9996       0.163244       0.947966       0.009549       0.160152   \n",
       "9997       0.008939       0.132476       0.001972       0.007586   \n",
       "9998       0.005045       0.002088       0.000410       0.005311   \n",
       "9999       0.013113       0.006877       0.002075       0.018499   \n",
       "\n",
       "      smbert_prob_8  \n",
       "0          0.786138  \n",
       "1          0.727966  \n",
       "2          0.742522  \n",
       "3          0.276741  \n",
       "4          0.201762  \n",
       "...             ...  \n",
       "9995       0.278770  \n",
       "9996       0.109565  \n",
       "9997       0.003270  \n",
       "9998       0.002019  \n",
       "9999       0.017215  \n",
       "\n",
       "[10000 rows x 9 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train_prob_df=pd.DataFrame(np.hstack(tmp_train),columns=[f'smbert_prob_{i}' for i in range(18)])\n",
    "# test_prob_df=pd.DataFrame(np.hstack(tmp_test),columns=[f'smbert_prob_{i}' for i in range(18)])\n",
    "# prob_df=pd.concat([train_prob_df,test_prob_df],axis=0).reset_index(drop=True)\n",
    "# prob_df\n",
    "\n",
    "train_prob_df=pd.DataFrame(np.vstack(tmp_train).T,columns=[f'smbert_prob_{i}' for i in range(9)])\n",
    "test_prob_df=pd.DataFrame(np.vstack(tmp_test).T,columns=[f'smbert_prob_{i}' for i in range(9)])\n",
    "prob_df=pd.concat([train_prob_df,test_prob_df],axis=0).reset_index(drop=True)\n",
    "prob_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.concat([data,prob_df],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['q1', 'q2', 'label', 'q1_len', 'q2_len', 'q1q2_len_diff',\n",
       "       'q1q2_len_diff_abs', 'q1q2_rate', 'q2q1_rate', 'q1_end_special',\n",
       "       ...\n",
       "       'sb_vec_sqeuclidean', 'smbert_prob_0', 'smbert_prob_1', 'smbert_prob_2',\n",
       "       'smbert_prob_3', 'smbert_prob_4', 'smbert_prob_5', 'smbert_prob_6',\n",
       "       'smbert_prob_7', 'smbert_prob_8'],\n",
       "      dtype='object', length=342)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "335"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_feas=['q1','q2','label','q1_words_list','q2_words_list','query','title']\n",
    "features=[col for col in data.columns if col not in no_feas]\n",
    "\n",
    "train,test=data[:train_size],data[train_size:]\n",
    "len(features)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train[features] # 训练集输入\n",
    "y = train['label'] # 训练集标签\n",
    "X_test = test[features] # 测试集输入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import lightgbm as lgb\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_fold = 10\n",
    "folds = KFold(n_splits=n_fold, shuffle=True,random_state=1314)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 200 rounds\n",
      "[50]\ttraining's binary_logloss: 0.196014\tvalid_1's binary_logloss: 0.232631\n",
      "[100]\ttraining's binary_logloss: 0.167501\tvalid_1's binary_logloss: 0.23384\n",
      "[150]\ttraining's binary_logloss: 0.146128\tvalid_1's binary_logloss: 0.233459\n",
      "[200]\ttraining's binary_logloss: 0.12881\tvalid_1's binary_logloss: 0.235922\n",
      "Early stopping, best iteration is:\n",
      "[40]\ttraining's binary_logloss: 0.203593\tvalid_1's binary_logloss: 0.23001\n",
      "Training until validation scores don't improve for 200 rounds\n",
      "[50]\ttraining's binary_logloss: 0.198043\tvalid_1's binary_logloss: 0.210849\n",
      "[100]\ttraining's binary_logloss: 0.168977\tvalid_1's binary_logloss: 0.213345\n",
      "[150]\ttraining's binary_logloss: 0.148068\tvalid_1's binary_logloss: 0.217567\n",
      "[200]\ttraining's binary_logloss: 0.131482\tvalid_1's binary_logloss: 0.220158\n",
      "[250]\ttraining's binary_logloss: 0.116575\tvalid_1's binary_logloss: 0.224799\n",
      "Early stopping, best iteration is:\n",
      "[55]\ttraining's binary_logloss: 0.194567\tvalid_1's binary_logloss: 0.209682\n",
      "Training until validation scores don't improve for 200 rounds\n",
      "[50]\ttraining's binary_logloss: 0.19852\tvalid_1's binary_logloss: 0.221588\n",
      "[100]\ttraining's binary_logloss: 0.170768\tvalid_1's binary_logloss: 0.215706\n",
      "[150]\ttraining's binary_logloss: 0.15012\tvalid_1's binary_logloss: 0.21798\n",
      "[200]\ttraining's binary_logloss: 0.132497\tvalid_1's binary_logloss: 0.220126\n",
      "[250]\ttraining's binary_logloss: 0.118188\tvalid_1's binary_logloss: 0.224038\n",
      "[300]\ttraining's binary_logloss: 0.105675\tvalid_1's binary_logloss: 0.224185\n",
      "Early stopping, best iteration is:\n",
      "[135]\ttraining's binary_logloss: 0.156059\tvalid_1's binary_logloss: 0.21509\n",
      "Training until validation scores don't improve for 200 rounds\n",
      "[50]\ttraining's binary_logloss: 0.195996\tvalid_1's binary_logloss: 0.233247\n",
      "[100]\ttraining's binary_logloss: 0.168817\tvalid_1's binary_logloss: 0.232822\n",
      "[150]\ttraining's binary_logloss: 0.147906\tvalid_1's binary_logloss: 0.230059\n",
      "[200]\ttraining's binary_logloss: 0.130955\tvalid_1's binary_logloss: 0.232831\n",
      "[250]\ttraining's binary_logloss: 0.116874\tvalid_1's binary_logloss: 0.23323\n",
      "[300]\ttraining's binary_logloss: 0.10436\tvalid_1's binary_logloss: 0.234234\n",
      "Early stopping, best iteration is:\n",
      "[145]\ttraining's binary_logloss: 0.149793\tvalid_1's binary_logloss: 0.229432\n",
      "Training until validation scores don't improve for 200 rounds\n",
      "[50]\ttraining's binary_logloss: 0.197379\tvalid_1's binary_logloss: 0.22243\n",
      "[100]\ttraining's binary_logloss: 0.169716\tvalid_1's binary_logloss: 0.222404\n",
      "[150]\ttraining's binary_logloss: 0.14826\tvalid_1's binary_logloss: 0.222839\n",
      "[200]\ttraining's binary_logloss: 0.130793\tvalid_1's binary_logloss: 0.224923\n",
      "[250]\ttraining's binary_logloss: 0.116622\tvalid_1's binary_logloss: 0.227817\n",
      "Early stopping, best iteration is:\n",
      "[69]\ttraining's binary_logloss: 0.185893\tvalid_1's binary_logloss: 0.219028\n",
      "Training until validation scores don't improve for 200 rounds\n",
      "[50]\ttraining's binary_logloss: 0.198677\tvalid_1's binary_logloss: 0.205083\n",
      "[100]\ttraining's binary_logloss: 0.17117\tvalid_1's binary_logloss: 0.209658\n",
      "[150]\ttraining's binary_logloss: 0.150254\tvalid_1's binary_logloss: 0.213614\n",
      "[200]\ttraining's binary_logloss: 0.132769\tvalid_1's binary_logloss: 0.213808\n",
      "Early stopping, best iteration is:\n",
      "[47]\ttraining's binary_logloss: 0.200923\tvalid_1's binary_logloss: 0.204173\n",
      "Training until validation scores don't improve for 200 rounds\n",
      "[50]\ttraining's binary_logloss: 0.198381\tvalid_1's binary_logloss: 0.202431\n",
      "[100]\ttraining's binary_logloss: 0.169815\tvalid_1's binary_logloss: 0.20388\n",
      "[150]\ttraining's binary_logloss: 0.149669\tvalid_1's binary_logloss: 0.205668\n",
      "[200]\ttraining's binary_logloss: 0.132335\tvalid_1's binary_logloss: 0.207503\n",
      "[250]\ttraining's binary_logloss: 0.118187\tvalid_1's binary_logloss: 0.209977\n",
      "Early stopping, best iteration is:\n",
      "[53]\ttraining's binary_logloss: 0.196134\tvalid_1's binary_logloss: 0.202188\n",
      "Training until validation scores don't improve for 200 rounds\n",
      "[50]\ttraining's binary_logloss: 0.200717\tvalid_1's binary_logloss: 0.187129\n",
      "[100]\ttraining's binary_logloss: 0.172519\tvalid_1's binary_logloss: 0.184411\n",
      "[150]\ttraining's binary_logloss: 0.151016\tvalid_1's binary_logloss: 0.182873\n",
      "[200]\ttraining's binary_logloss: 0.134007\tvalid_1's binary_logloss: 0.181429\n",
      "[250]\ttraining's binary_logloss: 0.119114\tvalid_1's binary_logloss: 0.182547\n",
      "[300]\ttraining's binary_logloss: 0.106582\tvalid_1's binary_logloss: 0.183519\n",
      "[350]\ttraining's binary_logloss: 0.0959709\tvalid_1's binary_logloss: 0.188027\n",
      "[400]\ttraining's binary_logloss: 0.0865751\tvalid_1's binary_logloss: 0.190527\n",
      "Early stopping, best iteration is:\n",
      "[203]\ttraining's binary_logloss: 0.133098\tvalid_1's binary_logloss: 0.18111\n",
      "Training until validation scores don't improve for 200 rounds\n",
      "[50]\ttraining's binary_logloss: 0.1919\tvalid_1's binary_logloss: 0.26593\n",
      "[100]\ttraining's binary_logloss: 0.163134\tvalid_1's binary_logloss: 0.279908\n",
      "[150]\ttraining's binary_logloss: 0.142382\tvalid_1's binary_logloss: 0.284437\n",
      "[200]\ttraining's binary_logloss: 0.125029\tvalid_1's binary_logloss: 0.290758\n",
      "Early stopping, best iteration is:\n",
      "[32]\ttraining's binary_logloss: 0.208954\tvalid_1's binary_logloss: 0.261636\n",
      "Training until validation scores don't improve for 200 rounds\n",
      "[50]\ttraining's binary_logloss: 0.195551\tvalid_1's binary_logloss: 0.230965\n",
      "[100]\ttraining's binary_logloss: 0.167442\tvalid_1's binary_logloss: 0.23262\n",
      "[150]\ttraining's binary_logloss: 0.146157\tvalid_1's binary_logloss: 0.238331\n",
      "[200]\ttraining's binary_logloss: 0.128468\tvalid_1's binary_logloss: 0.241635\n",
      "Early stopping, best iteration is:\n",
      "[41]\ttraining's binary_logloss: 0.202356\tvalid_1's binary_logloss: 0.230937\n"
     ]
    }
   ],
   "source": [
    "params = {\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'binary',\n",
    "    'num_leaves': 5,\n",
    "    'max_depth': 6,\n",
    "    'min_data_in_leaf': 450,\n",
    "    'learning_rate': 0.1,\n",
    "#     'learning_rate': 0.1,\n",
    "\n",
    "    'feature_fraction': 0.9,\n",
    "    'bagging_fraction': 0.95,\n",
    "    'bagging_freq': 5,\n",
    "    'lambda_l1': 1,  \n",
    "    'lambda_l2': 0.001,  # 越小l2正则程度越高\n",
    "    'min_gain_to_split': 0.2,\n",
    "}\n",
    "feat_imp_df = pd.DataFrame({'feat': features, 'imp': 0}) \n",
    "oof = np.zeros(len(X))\n",
    "prediction = np.zeros(len(X_test))\n",
    "for fold_n, (train_index, valid_index) in enumerate(folds.split(X)):\n",
    "    X_train, X_valid = X[features].iloc[train_index], X[features].iloc[valid_index]\n",
    "    y_train, y_valid = y[train_index], y[valid_index]\n",
    "    model = model = lgb.LGBMClassifier(**params, n_estimators=50000, n_jobs=-1)\n",
    "    model.fit(X_train, y_train,\n",
    "              eval_set=[(X_train, y_train), (X_valid, y_valid)],\n",
    "              eval_metric='binary_logloss',\n",
    "              verbose=50, early_stopping_rounds=200)\n",
    "    y_pred_valid = model.predict(X_valid)\n",
    "    y_pred = model.predict(X_test, num_iteration=model.best_iteration_)\n",
    "    feat_imp_df['imp'] += model.feature_importances_ / 5\n",
    "    oof[valid_index] = y_pred_valid.reshape(-1, )\n",
    "    prediction += y_pred\n",
    "prediction /= n_fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feat</th>\n",
       "      <th>imp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>332</th>\n",
       "      <td>smbert_prob_6</td>\n",
       "      <td>69.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>331</th>\n",
       "      <td>smbert_prob_5</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327</th>\n",
       "      <td>smbert_prob_1</td>\n",
       "      <td>48.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>334</th>\n",
       "      <td>smbert_prob_8</td>\n",
       "      <td>46.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>329</th>\n",
       "      <td>smbert_prob_3</td>\n",
       "      <td>21.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>330</th>\n",
       "      <td>smbert_prob_4</td>\n",
       "      <td>17.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333</th>\n",
       "      <td>smbert_prob_7</td>\n",
       "      <td>12.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>q1_pos_5</td>\n",
       "      <td>9.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317</th>\n",
       "      <td>sb_sb_vec_euclidean</td>\n",
       "      <td>7.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>q1_vec_57</td>\n",
       "      <td>6.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295</th>\n",
       "      <td>fuzz_qratio</td>\n",
       "      <td>6.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326</th>\n",
       "      <td>smbert_prob_0</td>\n",
       "      <td>6.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>q2_vec_37</td>\n",
       "      <td>6.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>q1_vec_67</td>\n",
       "      <td>6.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>fuzz_token_set_ratio</td>\n",
       "      <td>6.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>q2_vec_91</td>\n",
       "      <td>5.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>328</th>\n",
       "      <td>smbert_prob_2</td>\n",
       "      <td>5.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>q2_vec_27</td>\n",
       "      <td>5.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>q1_vec_60</td>\n",
       "      <td>5.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>q2_vec_44</td>\n",
       "      <td>5.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278</th>\n",
       "      <td>WordMatchShare_query</td>\n",
       "      <td>4.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>q1_vec_48</td>\n",
       "      <td>4.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>q1_vec_16</td>\n",
       "      <td>4.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257</th>\n",
       "      <td>q2_vec_94</td>\n",
       "      <td>4.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221</th>\n",
       "      <td>q2_vec_58</td>\n",
       "      <td>4.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>fuzz_partial_token_sort_ratio</td>\n",
       "      <td>4.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>q2_vec_24</td>\n",
       "      <td>4.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>vec_chebyshev</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>q1_vec_92</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>fuzz_token_sort_ratio</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              feat   imp\n",
       "332                  smbert_prob_6  69.6\n",
       "331                  smbert_prob_5  50.0\n",
       "327                  smbert_prob_1  48.2\n",
       "334                  smbert_prob_8  46.2\n",
       "329                  smbert_prob_3  21.6\n",
       "330                  smbert_prob_4  17.6\n",
       "333                  smbert_prob_7  12.8\n",
       "13                        q1_pos_5   9.8\n",
       "317            sb_sb_vec_euclidean   7.4\n",
       "120                      q1_vec_57   6.8\n",
       "295                    fuzz_qratio   6.8\n",
       "326                  smbert_prob_0   6.8\n",
       "200                      q2_vec_37   6.4\n",
       "130                      q1_vec_67   6.2\n",
       "300           fuzz_token_set_ratio   6.2\n",
       "254                      q2_vec_91   5.6\n",
       "328                  smbert_prob_2   5.4\n",
       "190                      q2_vec_27   5.2\n",
       "123                      q1_vec_60   5.2\n",
       "207                      q2_vec_44   5.2\n",
       "278           WordMatchShare_query   4.8\n",
       "111                      q1_vec_48   4.8\n",
       "79                       q1_vec_16   4.8\n",
       "257                      q2_vec_94   4.8\n",
       "221                      q2_vec_58   4.6\n",
       "299  fuzz_partial_token_sort_ratio   4.2\n",
       "187                      q2_vec_24   4.2\n",
       "58                   vec_chebyshev   4.0\n",
       "155                      q1_vec_92   4.0\n",
       "301          fuzz_token_sort_ratio   4.0"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feat_imp_df.sort_values(by='imp',ascending=False)[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.909"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "y_pred = (oof > 0.5).astype(int)\n",
    "# score=accuracy_score(np.round(abs(oof)) ,train['label'].values)\n",
    "score=accuracy_score(y_pred ,train['label'].values)\n",
    "\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_pred = (prediction > 0.5).astype(int)\n",
    "sample_submit['label']=sub_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_submit[['label']].to_csv('lgb.csv',index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    2929\n",
       "0    2071\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_submit['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 优化：\n",
    "# 0 base 839 8406\n",
    "# 1.word2vec iter=5改为iter=10 8406 8508\n",
    "# 2.加了tfidf之后 0.8496 0.848 降分\n",
    "# 3. 词粒度特征  0.8628\n",
    "# 4. 直接添加词向量特征 0.8588 0.8608 降分\n",
    "# 5.sb_sim 0.8708\n",
    "# 6.tfidf_sim 0.8756 0.8752\n",
    "# 7.加入bert prob 0.9096 0.9076\n",
    "# 8.bert 0.907\n",
    "# 9.bert单概率输出以及去除word2vec向量特征 0.9104\n",
    "# 10. reg改成cls 0.9112 9078\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}