check2_process.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import re\n",
    "\n",
    "import scipy\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from Bio import SeqIO\n",
    "\n",
    "from helperFxns import lett2num, alg2bin, filterAln, simMat, num2lett\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After filtering, the number of filtered sequences for file A.fasta: 3104\n",
      "After filtering, the number of filtered positions for file A.fasta: 161\n",
      "After filtering, the number of filtered sequences for file B.fasta: 3040\n",
      "After filtering, the number of filtered positions for file B.fasta: 264\n",
      "After filtering, the number of filtered sequences for file C.fasta: 3178\n",
      "After filtering, the number of filtered positions for file C.fasta: 267\n",
      "After filtering, the number of filtered sequences for file D.fasta: 3643\n",
      "After filtering, the number of filtered positions for file D.fasta: 401\n"
     ]
    }
   ],
   "source": [
    "# if it is one of A.fasta, B.fasta, C.fasta, D.fasta, then it is the raw MSA file\n",
    "# use regular expression to match the file name\n",
    "\n",
    "files = [f for f in os.listdir() if re.match(r'[A-D]\\.fasta', f)]\n",
    "\n",
    "for file in files:\n",
    "    # read the file with the Bio.SeqIO module\n",
    "    encoding_file = file.split('.')[0] + '_one-hot.npy'\n",
    "    header_file = file.split('.')[0] + '_header.txt'\n",
    "    filter_header_file = file.split('.')[0] + '_filter_header.txt'\n",
    "    headers = []\n",
    "    seqs = []\n",
    "    for seq_record in SeqIO.parse(file, \"fasta\"):\n",
    "        header = '>' + str(seq_record.description)\n",
    "        headers.append(header)\n",
    "        seq = str(seq_record.seq)\n",
    "        seqs.append(seq)\n",
    "\n",
    "\n",
    "    with open(header_file, 'w') as f:\n",
    "        for header in headers:\n",
    "            f.write(header)\n",
    "            f.write('\\n')\n",
    "\n",
    "    # one-hot encode the sequences\n",
    "    num = lett2num(seqs, code='ACDEFGHIKLMNPQRSTVWY-')\n",
    "    # print(num)\n",
    "    bin = alg2bin(num, N_aa=21)\n",
    "    # print(bin)\n",
    "    # sys.exit()\n",
    "\n",
    "    # filter out highly gapped positions and sequences according to the cutoff\n",
    "    hdFilter, seqFilter = filterAln(headers, num)\n",
    "    # print(seqFilter)\n",
    "    # print(seqFilter.shape)\n",
    "    print(\"After filtering, the number of filtered sequences for file {}: {}\".format(file, seqFilter.shape[0]))\n",
    "    print(\"After filtering, the number of filtered positions for file {}: {}\".format(file, seqFilter.shape[1]))\n",
    "    # print(len(hdFilter))\n",
    "\n",
    "    with open(filter_header_file, 'w') as f:\n",
    "        for header in hdFilter:\n",
    "            f.write(header)\n",
    "            f.write('\\n')\n",
    "    # print(seqFilter)\n",
    "    filtered_seq_list = num2lett(seqFilter, code='ACDEFGHIKLMNPQRSTVWY-')\n",
    "    # print(hdFilter)\n",
    "    # print(filtered_seq_list)\n",
    "    filtered_fasta_file = file.split('.')[0] + '_filtered.fasta'\n",
    "\n",
    "\n",
    "    if len(hdFilter) == len(filtered_seq_list):\n",
    "        with open(filtered_fasta_file, 'w') as f:\n",
    "            for i in range(len(hdFilter)):\n",
    "                f.write(hdFilter[i])\n",
    "                f.write('\\n')\n",
    "                f.write(filtered_seq_list[i])\n",
    "                f.write('\\n')\n",
    "    else:\n",
    "        raise ValueError(\"The number of headers and sequences do not match, please check the filtering process\")\n",
    "\n",
    "    # filter_bin = alg2bin(seqFilter, N_aa=21)\n",
    "\n",
    "    # # compute a sequence identity matrix\n",
    "    # sim = simMat(filter_bin, seqFilter.shape[1])\n",
    "\n",
    "    # # sim is a symmetric matrix, get the values above the diagonal\n",
    "    # sim_diag = sim[np.triu_indices(sim.shape[0], k=1)]\n",
    "    # # convert to a shape (n,) array\n",
    "    # sim_diag = np.array(sim_diag).reshape(-1)\n",
    "    \n",
    "    # # calculate the mean of the sequence identities\n",
    "    # mean_sim = np.mean(sim_diag)\n",
    "    # print(\"The mean sequence identity for file {}: {}\".format(file, round(mean_sim, 3)))\n",
    "\n",
    "    # # plot the histogram of sequence identities for each MSA\n",
    "    # plt.figure()\n",
    "    # plt.hist(sim_diag, bins=100)\n",
    "    # plt.xlabel('Sequence identity')\n",
    "    # plt.ylabel('Counts')\n",
    "    # plt.title('Histogram of sequence identities of proteins in MSA_{}'.format(\n",
    "    #     file.split('.')[0]))\n",
    "    # # plt.show()\n",
    "    # plt.savefig(file.split('.')[0] + '_hist.png')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['A_filtered.fasta', 'B_filtered.fasta', 'C_filtered.fasta', 'D_filtered.fasta']\n",
      "Tax ID 1513468_0 is duplicated\n",
      "Tax ID 2589817_0 is duplicated\n",
      "Tax ID 2027290_0 is duplicated\n",
      "Tax ID 565_0 is duplicated\n",
      "Tax ID 2041079_0 is duplicated\n",
      "Tax ID 1736699_0 is duplicated\n",
      "Tax ID 208224_0 is duplicated\n",
      "Tax ID 2027290_0 is duplicated\n",
      "Tax ID 381306_0 is duplicated\n",
      "Tax ID 171383_0 is duplicated\n",
      "Tax ID 1871111_0 is duplicated\n",
      "Tax ID 2136183_0 is duplicated\n",
      "Tax ID 1443941_0 is duplicated\n",
      "Tax ID 2004646_0 is duplicated\n",
      "Tax ID 300_0 is duplicated\n",
      "Tax ID 2479853_0 is duplicated\n",
      "Tax ID 1736226_0 is duplicated\n",
      "Tax ID 2060418_0 is duplicated\n",
      "Tax ID 2293832_0 is duplicated\n",
      "Tax ID 1869229_0 is duplicated\n",
      "Tax ID 216595_0 is duplicated\n",
      "Tax ID 2054919_0 is duplicated\n",
      "Tax ID 33069_8 is duplicated\n",
      "Tax ID 255519_0 is duplicated\n",
      "Tax ID 565_0 is duplicated\n",
      "Tax ID 1985876_0 is duplicated\n",
      "Tax ID 1736699_0 is duplicated\n",
      "Tax ID 83655_0 is duplicated\n",
      "Tax ID 1812935_0 is duplicated\n",
      "Tax ID 2041079_0 is duplicated\n",
      "Tax ID 208224_0 is duplicated\n",
      "Tax ID 158836_0 is duplicated\n",
      "Tax ID 1398493_0 is duplicated\n",
      "Tax ID 571_1 is duplicated\n",
      "Tax ID 571_3 is duplicated\n",
      "Tax ID 57706_0 is duplicated\n",
      "Tax ID 587_0 is duplicated\n",
      "Tax ID 2027290_0 is duplicated\n",
      "Tax ID 368603_0 is duplicated\n",
      "Tax ID 582_2 is duplicated\n",
      "Tax ID 584_0 is duplicated\n",
      "Tax ID 1851571_0 is duplicated\n",
      "Tax ID 1427518_0 is duplicated\n",
      "Tax ID 880157_0 is duplicated\n",
      "Tax ID 742178_0 is duplicated\n",
      "Tax ID 1513468_0 is duplicated\n",
      "Tax ID 932213_0 is duplicated\n",
      "Tax ID 2233853_0 is duplicated\n",
      "Tax ID 2013866_0 is duplicated\n",
      "Tax ID 332186_0 is duplicated\n",
      "Tax ID 2058327_0 is duplicated\n",
      "Tax ID 80854_0 is duplicated\n",
      "Tax ID 756698_0 is duplicated\n",
      "Tax ID 320778_0 is duplicated\n",
      "Tax ID 1194427_0 is duplicated\n",
      "Tax ID 212663_0 is duplicated\n",
      "Tax ID 2302434_0 is duplicated\n",
      "Tax ID 663_1 is duplicated\n",
      "Tax ID 680_0 is duplicated\n",
      "Tax ID 171383_0 is duplicated\n",
      "Tax ID 2077090_0 is duplicated\n",
      "Tax ID 1159017_0 is duplicated\n",
      "Tax ID 633417_0 is duplicated\n",
      "Tax ID 1201293_0 is duplicated\n",
      "Tax ID 672_0 is duplicated\n",
      "Tax ID 641665_0 is duplicated\n",
      "Tax ID 2057807_0 is duplicated\n",
      "Tax ID 2545632_0 is duplicated\n",
      "Tax ID 555778_0 is duplicated\n",
      "Tax ID 454594_0 is duplicated\n",
      "Tax ID 1122206_0 is duplicated\n",
      "Tax ID 2589817_0 is duplicated\n",
      "Tax ID 2601894_0 is duplicated\n",
      "Tax ID 1822214_0 is duplicated\n",
      "Tax ID 2547392_0 is duplicated\n",
      "Tax ID 1822233_0 is duplicated\n",
      "Tax ID 1903177_0 is duplicated\n",
      "Tax ID 2545719_0 is duplicated\n",
      "Tax ID 675524_0 is duplicated\n",
      "Tax ID 1078846_0 is duplicated\n",
      "Tax ID 1985301_0 is duplicated\n",
      "Tax ID 1912599_0 is duplicated\n",
      "Tax ID 663_1 is duplicated\n",
      "Tax ID 1822233_0 is duplicated\n",
      "Tax ID 1736226_0 is duplicated\n",
      "Tax ID 294_18 is duplicated\n",
      "Tax ID 320778_0 is duplicated\n",
      "Tax ID 2304625_0 is duplicated\n",
      "Tax ID 132476_1 is duplicated\n",
      "Tax ID 1699310_0 is duplicated\n",
      "Tax ID 294_39 is duplicated\n",
      "Tax ID 251700_0 is duplicated\n",
      "Tax ID 294_27 is duplicated\n",
      "Tax ID 2058307_0 is duplicated\n",
      "Tax ID 1330531_0 is duplicated\n",
      "Tax ID 395599_0 is duplicated\n",
      "Tax ID 1981513_0 is duplicated\n",
      "Tax ID 990268_2 is duplicated\n",
      "Tax ID 675524_0 is duplicated\n",
      "Tax ID 2233853_0 is duplicated\n",
      "Tax ID 43657_2 is duplicated\n",
      "Tax ID 2599398_0 is duplicated\n",
      "Tax ID 2305248_0 is duplicated\n",
      "Tax ID 1158152_0 is duplicated\n",
      "Tax ID 300_1 is duplicated\n",
      "Tax ID 2545719_0 is duplicated\n",
      "Tax ID 314275_0 is duplicated\n",
      "Tax ID 568106_0 is duplicated\n",
      "Tax ID 1476959_0 is duplicated\n",
      "Tax ID 713586_0 is duplicated\n",
      "Tax ID 454594_0 is duplicated\n",
      "Tax ID 41202_0 is duplicated\n",
      "Tax ID 1078846_0 is duplicated\n",
      "Tax ID 1135990_0 is duplicated\n",
      "Tax ID 1403540_0 is duplicated\n",
      "Tax ID 1177159_0 is duplicated\n",
      "Tax ID 42565_0 is duplicated\n",
      "Tax ID 1981676_0 is duplicated\n",
      "Tax ID 2211402_0 is duplicated\n",
      "Tax ID 2033802_0 is duplicated\n",
      "Tax ID 316_6 is duplicated\n",
      "Tax ID 1323744_0 is duplicated\n",
      "Tax ID 1323663_0 is duplicated\n",
      "Tax ID 553151_0 is duplicated\n",
      "Tax ID 2268758_0 is duplicated\n",
      "Tax ID 2201898_0 is duplicated\n",
      "Tax ID 2484250_0 is duplicated\n",
      "Tax ID 2483111_0 is duplicated\n",
      "Tax ID 44933_0 is duplicated\n",
      "Tax ID 2587844_0 is duplicated\n",
      "Tax ID 284577_1 is duplicated\n",
      "Tax ID 2054178_0 is duplicated\n",
      "Tax ID 2587849_0 is duplicated\n",
      "Tax ID 416872_0 is duplicated\n",
      "Tax ID 490901_0 is duplicated\n",
      "Tax ID 1434831_0 is duplicated\n",
      "Tax ID 574966_0 is duplicated\n",
      "Tax ID 2498860_0 is duplicated\n",
      "Tax ID 2498859_0 is duplicated\n",
      "Tax ID 1176243_0 is duplicated\n",
      "Tax ID 2024405_0 is duplicated\n",
      "Tax ID 29570_1 is duplicated\n",
      "Tax ID 2585770_0 is duplicated\n",
      "Tax ID 115561_0 is duplicated\n",
      "Tax ID 29570_0 is duplicated\n",
      "Tax ID 1118153_0 is duplicated\n",
      "Tax ID 1761788_0 is duplicated\n",
      "Tax ID 1684425_0 is duplicated\n",
      "Tax ID 1975722_0 is duplicated\n",
      "Tax ID 1479235_0 is duplicated\n",
      "Tax ID 1666906_0 is duplicated\n",
      "Tax ID 416873_0 is duplicated\n",
      "Tax ID 31910_0 is duplicated\n",
      "Tax ID 1072583_0 is duplicated\n",
      "Tax ID 1123773_0 is duplicated\n",
      "Tax ID 1805819_0 is duplicated\n",
      "Tax ID 272774_0 is duplicated\n",
      "Tax ID 447567_0 is duplicated\n",
      "Tax ID 232346_0 is duplicated\n",
      "Tax ID 2042704_0 is duplicated\n",
      "Tax ID 1962264_0 is duplicated\n",
      "Tax ID 999141_0 is duplicated\n",
      "Tax ID 1971364_0 is duplicated\n",
      "Tax ID 44935_0 is duplicated\n",
      "Tax ID 213554_1 is duplicated\n",
      "Tax ID 2058313_0 is duplicated\n",
      "Tax ID 115553_0 is duplicated\n",
      "Tax ID 1504981_0 is duplicated\n",
      "Tax ID 1610576_0 is duplicated\n",
      "Tax ID 213554_0 is duplicated\n",
      "Tax ID 232346_1 is duplicated\n",
      "Tax ID 2058321_0 is duplicated\n",
      "Tax ID 1121943_0 is duplicated\n",
      "Tax ID 2032625_0 is duplicated\n",
      "Tax ID 590061_0 is duplicated\n",
      "Tax ID 419596_0 is duplicated\n",
      "Tax ID 2497861_0 is duplicated\n",
      "Tax ID 119000_0 is duplicated\n",
      "Tax ID 2081032_0 is duplicated\n",
      "Tax ID 657387_0 is duplicated\n",
      "Tax ID 1982043_0 is duplicated\n",
      "Tax ID 1949087_0 is duplicated\n",
      "Tax ID 1492922_0 is duplicated\n",
      "Tax ID 1771309_0 is duplicated\n",
      "Tax ID 184065_0 is duplicated\n",
      "Tax ID 1949082_0 is duplicated\n",
      "Tax ID 1267424_0 is duplicated\n",
      "Tax ID 1949081_0 is duplicated\n",
      "Tax ID 1928762_0 is duplicated\n",
      "Tax ID 1949085_0 is duplicated\n",
      "Tax ID 404433_0 is duplicated\n",
      "Tax ID 430457_0 is duplicated\n",
      "Tax ID 1882734_0 is duplicated\n",
      "Tax ID 487184_0 is duplicated\n",
      "Tax ID 1434072_0 is duplicated\n",
      "Tax ID 2078689_0 is duplicated\n",
      "Tax ID 1848729_0 is duplicated\n",
      "Tax ID 1981513_0 is duplicated\n",
      "Tax ID 2586906_0 is duplicated\n",
      "Tax ID 254161_0 is duplicated\n",
      "Tax ID 1708783_0 is duplicated\n",
      "Tax ID 857227_0 is duplicated\n",
      "Tax ID 857252_0 is duplicated\n",
      "Tax ID 472181_0 is duplicated\n",
      "Tax ID 2487347_0 is duplicated\n",
      "Tax ID 2508710_0 is duplicated\n",
      "Tax ID 42566_0 is duplicated\n",
      "Tax ID 768066_0 is duplicated\n",
      "Tax ID 2185141_0 is duplicated\n",
      "Tax ID 2603215_0 is duplicated\n",
      "Tax ID 2528979_0 is duplicated\n",
      "Tax ID 626887_0 is duplicated\n",
      "Tax ID 2505979_0 is duplicated\n",
      "Tax ID 2529849_0 is duplicated\n",
      "Tax ID 2529842_0 is duplicated\n",
      "Tax ID 2529841_0 is duplicated\n",
      "Tax ID 1977872_0 is duplicated\n",
      "Tax ID 1217715_0 is duplicated\n",
      "Tax ID 1209355_0 is duplicated\n",
      "Tax ID 1675902_0 is duplicated\n",
      "Tax ID 1120929_0 is duplicated\n",
      "Tax ID 1960940_0 is duplicated\n",
      "Tax ID 2004647_0 is duplicated\n",
      "Tax ID 2004646_0 is duplicated\n",
      "Tax ID 1879049_0 is duplicated\n",
      "Tax ID 1775557_0 is duplicated\n",
      "Tax ID 1242245_0 is duplicated\n",
      "Tax ID 1977873_0 is duplicated\n",
      "Tax ID 2170730_0 is duplicated\n",
      "Tax ID 2419603_0 is duplicated\n",
      "Tax ID 2529850_0 is duplicated\n",
      "Tax ID 2529837_0 is duplicated\n",
      "Tax ID 1120927_0 is duplicated\n",
      "Tax ID 1806892_0 is duplicated\n",
      "Tax ID 2529839_0 is duplicated\n",
      "Tax ID 1789224_0 is duplicated\n",
      "Tax ID 1871111_0 is duplicated\n",
      "Tax ID 2006115_1 is duplicated\n",
      "Tax ID 1775557_0 is duplicated\n",
      "Tax ID 2126983_0 is duplicated\n",
      "Tax ID 1120926_0 is duplicated\n",
      "Tax ID 179995_0 is duplicated\n",
      "Tax ID 1304900_0 is duplicated\n",
      "Tax ID 1897630_0 is duplicated\n",
      "Tax ID 141451_0 is duplicated\n",
      "Tax ID 1121103_0 is duplicated\n",
      "Tax ID 377629_0 is duplicated\n",
      "Tax ID 1056820_0 is duplicated\n",
      "Tax ID 1628392_0 is duplicated\n",
      "Tax ID 349521_0 is duplicated\n",
      "Tax ID 1336806_0 is duplicated\n",
      "Tax ID 1121935_0 is duplicated\n",
      "Tax ID 1926491_0 is duplicated\n",
      "Tax ID 1572645_0 is duplicated\n",
      "Tax ID 2561897_0 is duplicated\n",
      "Tax ID 1571224_0 is duplicated\n",
      "Tax ID 1031538_0 is duplicated\n",
      "Tax ID 1122599_0 is duplicated\n",
      "Tax ID 1045558_0 is duplicated\n",
      "Tax ID 619304_0 is duplicated\n",
      "Tax ID 1897630_0 is duplicated\n",
      "Tax ID 1122200_0 is duplicated\n",
      "Tax ID 1122202_0 is duplicated\n",
      "Tax ID 1821621_0 is duplicated\n",
      "Tax ID 1714300_0 is duplicated\n",
      "Tax ID 1902503_0 is duplicated\n",
      "Tax ID 314277_0 is duplicated\n",
      "Tax ID 579484_0 is duplicated\n",
      "Tax ID 1137284_0 is duplicated\n",
      "Tax ID 28254_0 is duplicated\n",
      "Tax ID 295068_0 is duplicated\n",
      "Tax ID 491947_0 is duplicated\n",
      "Tax ID 717774_0 is duplicated\n",
      "Tax ID 491954_0 is duplicated\n",
      "Tax ID 2529379_0 is duplicated\n",
      "Tax ID 491952_0 is duplicated\n",
      "Tax ID 491951_0 is duplicated\n",
      "Tax ID 1561203_0 is duplicated\n",
      "Tax ID 2530201_0 is duplicated\n",
      "Tax ID 1792290_0 is duplicated\n",
      "Tax ID 1122207_0 is duplicated\n",
      "Tax ID 1208321_0 is duplicated\n",
      "Tax ID 178399_0 is duplicated\n",
      "Tax ID 383750_0 is duplicated\n",
      "Tax ID 2202418_0 is duplicated\n",
      "Tax ID 400668_0 is duplicated\n",
      "Tax ID 491950_0 is duplicated\n",
      "Tax ID 2071621_0 is duplicated\n",
      "Tax ID 491948_0 is duplicated\n",
      "Tax ID 1122206_0 is duplicated\n",
      "Tax ID 485751_0 is duplicated\n",
      "Tax ID 67824_0 is duplicated\n",
      "Tax ID 667129_0 is duplicated\n",
      "Tax ID 2233853_0 is duplicated\n",
      "Tax ID 571_0 is duplicated\n",
      "Tax ID 571_3 is duplicated\n",
      "Tax ID 2579935_0 is duplicated\n",
      "Tax ID 1134687_1 is duplicated\n",
      "Tax ID 1384589_0 is duplicated\n",
      "Tax ID 546_0 is duplicated\n",
      "Tax ID 546_3 is duplicated\n",
      "Tax ID 35703_0 is duplicated\n",
      "Tax ID 67824_0 is duplicated\n",
      "Tax ID 73098_0 is duplicated\n",
      "Tax ID 2545719_0 is duplicated\n",
      "Tax ID 55209_0 is duplicated\n",
      "Tax ID 137584_0 is duplicated\n",
      "Tax ID 1120963_0 is duplicated\n",
      "Tax ID 571_3 is duplicated\n",
      "Tax ID 571_0 is duplicated\n",
      "Tax ID 73098_0 is duplicated\n",
      "Tax ID 67824_0 is duplicated\n",
      "Tax ID 1134687_1 is duplicated\n",
      "Tax ID 1384589_0 is duplicated\n",
      "Tax ID 546_3 is duplicated\n",
      "Tax ID 546_0 is duplicated\n",
      "Tax ID 35703_0 is duplicated\n",
      "Tax ID 67824_0 is duplicated\n",
      "Tax ID 667129_0 is duplicated\n",
      "Tax ID 1827396_0 is duplicated\n",
      "Tax ID 2059725_0 is duplicated\n",
      "Tax ID 2282215_0 is duplicated\n",
      "Tax ID 680279_0 is duplicated\n",
      "Tax ID 1331007_0 is duplicated\n",
      "Tax ID 2283141_0 is duplicated\n",
      "Tax ID 1120969_0 is duplicated\n",
      "Tax ID 1671868_0 is duplicated\n",
      "Tax ID 462991_0 is duplicated\n",
      "Tax ID 1243014_0 is duplicated\n",
      "Tax ID 1676683_0 is duplicated\n",
      "Tax ID 2303987_0 is duplicated\n",
      "Tax ID 266810_0 is duplicated\n",
      "Tax ID 1891207_0 is duplicated\n",
      "Tax ID 870968_0 is duplicated\n",
      "Tax ID 2572923_0 is duplicated\n",
      "Tax ID 29498_0 is duplicated\n",
      "Tax ID 265668_0 is duplicated\n",
      "Tax ID 2231649_0 is duplicated\n",
      "Tax ID 1381081_0 is duplicated\n",
      "Tax ID 754436_0 is duplicated\n",
      "Tax ID 2302434_0 is duplicated\n",
      "Tax ID 2485123_0 is duplicated\n",
      "Tax ID 1903952_0 is duplicated\n",
      "Tax ID 1775171_0 is duplicated\n",
      "Tax ID 1342794_0 is duplicated\n",
      "Tax ID 294936_0 is duplicated\n",
      "Tax ID 388278_0 is duplicated\n",
      "Tax ID 658445_0 is duplicated\n",
      "Tax ID 675812_0 is duplicated\n",
      "Tax ID 1796497_0 is duplicated\n",
      "Tax ID 1581149_0 is duplicated\n",
      "Tax ID 1891184_0 is duplicated\n",
      "Tax ID 1761909_0 is duplicated\n",
      "Tax ID 663_1 is duplicated\n",
      "Tax ID 663_2 is duplicated\n",
      "Tax ID 691_0 is duplicated\n",
      "Tax ID 670_0 is duplicated\n",
      "Tax ID 1004316_0 is duplicated\n",
      "Tax ID 693_0 is duplicated\n",
      "Tax ID 171383_0 is duplicated\n",
      "Tax ID 2267621_0 is duplicated\n",
      "Tax ID 28229_0 is duplicated\n",
      "Tax ID 89401_0 is duplicated\n",
      "Tax ID 2058305_0 is duplicated\n",
      "Tax ID 511062_0 is duplicated\n",
      "Tax ID 129578_0 is duplicated\n",
      "Tax ID 84158_0 is duplicated\n",
      "Tax ID 1458355_0 is duplicated\n",
      "Tax ID 2233853_0 is duplicated\n",
      "Tax ID 1917163_0 is duplicated\n",
      "Tax ID 2593656_0 is duplicated\n",
      "Tax ID 1278307_0 is duplicated\n",
      "Tax ID 1278312_0 is duplicated\n",
      "Tax ID 271099_0 is duplicated\n",
      "Tax ID 271096_0 is duplicated\n",
      "Tax ID 425104_0 is duplicated\n",
      "Tax ID 1121878_0 is duplicated\n",
      "Tax ID 2569538_0 is duplicated\n",
      "Tax ID 2569539_0 is duplicated\n",
      "Tax ID 718193_0 is duplicated\n",
      "Tax ID 1121879_0 is duplicated\n",
      "Tax ID 1122201_0 is duplicated\n",
      "Tax ID 1232683_0 is duplicated\n",
      "Tax ID 568106_0 is duplicated\n",
      "Tax ID 1476959_0 is duplicated\n",
      "Tax ID 1501432_0 is duplicated\n",
      "Tax ID 419596_0 is duplicated\n",
      "Tax ID 1042375_0 is duplicated\n",
      "Tax ID 267374_0 is duplicated\n",
      "Tax ID 1122198_0 is duplicated\n",
      "Tax ID 254161_0 is duplicated\n",
      "Tax ID 1708783_0 is duplicated\n",
      "Tax ID 857227_0 is duplicated\n",
      "Tax ID 857252_0 is duplicated\n",
      "Tax ID 1697053_0 is duplicated\n",
      "Tax ID 1212548_0 is duplicated\n",
      "Tax ID 316_7 is duplicated\n",
      "Tax ID 316_2 is duplicated\n",
      "Tax ID 271420_0 is duplicated\n",
      "Tax ID 1729726_0 is duplicated\n",
      "Tax ID 316_6 is duplicated\n",
      "Tax ID 440512_0 is duplicated\n",
      "Tax ID 316_9 is duplicated\n",
      "Tax ID 1176259_0 is duplicated\n",
      "Tax ID 412955_0 is duplicated\n",
      "Tax ID 1793966_0 is duplicated\n",
      "Tax ID 1245526_0 is duplicated\n",
      "Tax ID 1392877_0 is duplicated\n",
      "Tax ID 915471_0 is duplicated\n",
      "Tax ID 1135990_0 is duplicated\n",
      "Tax ID 1821274_0 is duplicated\n",
      "Tax ID 2033802_0 is duplicated\n",
      "Tax ID 198616_0 is duplicated\n",
      "Tax ID 521719_0 is duplicated\n",
      "Tax ID 706570_0 is duplicated\n",
      "Tax ID 316_1 is duplicated\n",
      "Tax ID 1439940_0 is duplicated\n",
      "Tax ID 1212549_0 is duplicated\n",
      "Tax ID 1981676_0 is duplicated\n",
      "Tax ID 2026782_2 is duplicated\n",
      "Tax ID 2026782_1 is duplicated\n",
      "Tax ID 2026782_6 is duplicated\n",
      "Tax ID 2026782_5 is duplicated\n",
      "Tax ID 2026782_0 is duplicated\n",
      "Tax ID 247633_0 is duplicated\n",
      "Tax ID 1123400_0 is duplicated\n",
      "Tax ID 1122200_0 is duplicated\n",
      "Tax ID 2561897_0 is duplicated\n",
      "Tax ID 1031538_0 is duplicated\n",
      "Tax ID 619304_0 is duplicated\n",
      "Tax ID 1045558_0 is duplicated\n",
      "Tax ID 1122599_0 is duplicated\n",
      "Tax ID 1897630_0 is duplicated\n",
      "Tax ID 1122202_0 is duplicated\n",
      "Tax ID 1121103_0 is duplicated\n",
      "Tax ID 2219697_0 is duplicated\n",
      "Tax ID 2505979_0 is duplicated\n",
      "Tax ID 1336806_0 is duplicated\n",
      "Tax ID 1120963_0 is duplicated\n",
      "Tax ID 2571175_0 is duplicated\n",
      "Tax ID 465721_0 is duplicated\n",
      "Tax ID 1803913_0 is duplicated\n",
      "Tax ID 485751_0 is duplicated\n",
      "Tax ID 137584_0 is duplicated\n",
      "Tax ID 179995_0 is duplicated\n",
      "Tax ID 1945855_0 is duplicated\n",
      "Tax ID 2545719_0 is duplicated\n",
      "Tax ID 316_6 is duplicated\n",
      "Tax ID 2529842_0 is duplicated\n",
      "Tax ID 2529841_0 is duplicated\n",
      "Tax ID 1977872_0 is duplicated\n",
      "Tax ID 2529849_0 is duplicated\n",
      "Tax ID 1217715_0 is duplicated\n",
      "Tax ID 1209355_0 is duplicated\n",
      "Tax ID 1977873_0 is duplicated\n",
      "Tax ID 2529850_0 is duplicated\n",
      "Tax ID 1775557_0 is duplicated\n",
      "Tax ID 2170730_0 is duplicated\n",
      "Tax ID 1806892_0 is duplicated\n",
      "Tax ID 2529837_0 is duplicated\n",
      "Tax ID 1120927_0 is duplicated\n",
      "Tax ID 2004646_0 is duplicated\n",
      "Tax ID 1120929_0 is duplicated\n",
      "Tax ID 1209355_0 is duplicated\n",
      "Tax ID 2419603_0 is duplicated\n",
      "Tax ID 1879049_0 is duplicated\n",
      "Tax ID 2529839_0 is duplicated\n",
      "Tax ID 2004647_0 is duplicated\n",
      "Tax ID 1960940_0 is duplicated\n",
      "Tax ID 1242245_0 is duplicated\n",
      "Tax ID 1120926_0 is duplicated\n",
      "Tax ID 2126983_0 is duplicated\n",
      "Tax ID 1871111_0 is duplicated\n",
      "Tax ID 2006115_1 is duplicated\n",
      "Tax ID 1775557_0 is duplicated\n",
      "Tax ID 1982043_0 is duplicated\n",
      "Tax ID 1789224_0 is duplicated\n",
      "Tax ID 1262585_0 is duplicated\n",
      "Tax ID 281375_0 is duplicated\n",
      "Tax ID 1096779_0 is duplicated\n",
      "Tax ID 1582270_0 is duplicated\n",
      "Tax ID 1120926_0 is duplicated\n",
      "Tax ID 2126983_0 is duplicated\n",
      "Tax ID 2006115_1 is duplicated\n",
      "Tax ID 1871111_0 is duplicated\n",
      "Tax ID 1775557_0 is duplicated\n",
      "Tax ID 2233853_0 is duplicated\n",
      "Tax ID 1341683_0 is duplicated\n",
      "Tax ID 1217714_0 is duplicated\n",
      "Tax ID 2529845_0 is duplicated\n",
      "Tax ID 396323_0 is duplicated\n",
      "Tax ID 2069438_0 is duplicated\n",
      "Tax ID 2529849_0 is duplicated\n",
      "Tax ID 1977880_0 is duplicated\n",
      "Tax ID 1675902_0 is duplicated\n",
      "Tax ID 1217674_0 is duplicated\n",
      "Tax ID 1530123_0 is duplicated\n",
      "Tax ID 470_0 is duplicated\n",
      "Tax ID 903915_0 is duplicated\n",
      "Tax ID 471_1 is duplicated\n",
      "Tax ID 1785128_0 is duplicated\n",
      "Tax ID 1217688_0 is duplicated\n",
      "Tax ID 1217691_0 is duplicated\n",
      "Tax ID 871585_0 is duplicated\n",
      "Tax ID 436717_0 is duplicated\n",
      "Tax ID 1217653_0 is duplicated\n",
      "Tax ID 346222_0 is duplicated\n",
      "Tax ID 2060938_0 is duplicated\n",
      "Tax ID 40215_0 is duplicated\n",
      "Tax ID 348388_0 is duplicated\n",
      "Tax ID 1120928_0 is duplicated\n",
      "Tax ID 1217697_0 is duplicated\n",
      "Tax ID 1217708_0 is duplicated\n",
      "Tax ID 1561217_0 is duplicated\n",
      "Tax ID 1191460_0 is duplicated\n",
      "Tax ID 280145_0 is duplicated\n",
      "Tax ID 1217707_0 is duplicated\n",
      "Tax ID 134534_0 is duplicated\n",
      "Tax ID 1217700_0 is duplicated\n",
      "Tax ID 1217649_0 is duplicated\n",
      "Tax ID 1217706_0 is duplicated\n",
      "Tax ID 1217703_0 is duplicated\n",
      "Tax ID 1217699_0 is duplicated\n",
      "Tax ID 1776741_0 is duplicated\n",
      "Tax ID 1144673_0 is duplicated\n",
      "Tax ID 28090_0 is duplicated\n",
      "Tax ID 1217709_0 is duplicated\n",
      "Tax ID 70346_0 is duplicated\n",
      "Tax ID 1977876_0 is duplicated\n",
      "Tax ID 1509403_0 is duplicated\n",
      "Tax ID 1324350_0 is duplicated\n",
      "Tax ID 1608473_0 is duplicated\n",
      "Tax ID 1808001_0 is duplicated\n",
      "Tax ID 1341679_0 is duplicated\n",
      "Tax ID 1120929_0 is duplicated\n",
      "Tax ID 1209355_0 is duplicated\n",
      "Tax ID 1226327_0 is duplicated\n",
      "Tax ID 202954_0 is duplicated\n",
      "Tax ID 1120927_0 is duplicated\n",
      "Tax ID 2576829_0 is duplicated\n",
      "Tax ID 1977881_0 is duplicated\n",
      "Tax ID 1217715_0 is duplicated\n",
      "Tax ID 1805634_0 is duplicated\n",
      "Tax ID 2529842_0 is duplicated\n",
      "Tax ID 2529841_0 is duplicated\n",
      "Tax ID 1977875_0 is duplicated\n",
      "Tax ID 1977882_0 is duplicated\n",
      "Tax ID 1977883_0 is duplicated\n",
      "Tax ID 1977879_0 is duplicated\n",
      "Tax ID 2170730_0 is duplicated\n",
      "Tax ID 2529837_0 is duplicated\n",
      "Tax ID 2529850_0 is duplicated\n",
      "Tax ID 1242245_0 is duplicated\n",
      "Tax ID 2529839_0 is duplicated\n",
      "Tax ID 2293834_0 is duplicated\n",
      "Tax ID 2004646_0 is duplicated\n",
      "Tax ID 1879049_0 is duplicated\n",
      "Tax ID 2136183_0 is duplicated\n",
      "Tax ID 1839785_0 is duplicated\n",
      "Tax ID 1977874_0 is duplicated\n",
      "Tax ID 1029823_0 is duplicated\n",
      "Tax ID 2006115_0 is duplicated\n",
      "Tax ID 2419601_0 is duplicated\n",
      "Tax ID 981324_0 is duplicated\n",
      "Tax ID 2004647_0 is duplicated\n",
      "Tax ID 1736444_0 is duplicated\n",
      "Tax ID 1806892_0 is duplicated\n",
      "Tax ID 1879050_0 is duplicated\n",
      "Tax ID 1120925_0 is duplicated\n",
      "Tax ID 2419605_0 is duplicated\n",
      "Tax ID 2293835_0 is duplicated\n",
      "Tax ID 2419603_0 is duplicated\n",
      "Tax ID 1960940_0 is duplicated\n",
      "Tax ID 1977872_0 is duplicated\n",
      "Tax ID 1891224_0 is duplicated\n",
      "Tax ID 1443941_0 is duplicated\n",
      "Tax ID 1977877_0 is duplicated\n",
      "Tax ID 1977873_0 is duplicated\n",
      "Tax ID 2081032_0 is duplicated\n",
      "Tax ID 1949087_0 is duplicated\n",
      "Tax ID 1492922_0 is duplicated\n",
      "Tax ID 1771309_0 is duplicated\n",
      "Tax ID 430457_0 is duplicated\n",
      "Tax ID 1882734_0 is duplicated\n",
      "Tax ID 1949081_0 is duplicated\n",
      "Tax ID 768066_0 is duplicated\n",
      "Tax ID 2306583_0 is duplicated\n",
      "Tax ID 507626_0 is duplicated\n",
      "Tax ID 2282311_0 is duplicated\n",
      "Tax ID 2282305_0 is duplicated\n",
      "Tax ID 490901_0 is duplicated\n",
      "Tax ID 370769_0 is duplicated\n",
      "Tax ID 2054178_0 is duplicated\n",
      "Tax ID 2587849_0 is duplicated\n",
      "Tax ID 376427_0 is duplicated\n",
      "Tax ID 2599398_0 is duplicated\n",
      "Tax ID 419596_0 is duplicated\n",
      "Tax ID 590061_0 is duplicated\n",
      "Tax ID 77098_0 is duplicated\n",
      "Tax ID 1479235_0 is duplicated\n",
      "Tax ID 2058321_0 is duplicated\n",
      "Tax ID 1504981_0 is duplicated\n",
      "Tax ID 2024405_0 is duplicated\n",
      "Tax ID 1434831_0 is duplicated\n",
      "Tax ID 1176243_0 is duplicated\n",
      "Tax ID 574966_0 is duplicated\n",
      "Tax ID 2498860_0 is duplicated\n",
      "Tax ID 2498859_0 is duplicated\n",
      "Tax ID 1816686_0 is duplicated\n",
      "Tax ID 1761788_0 is duplicated\n",
      "Tax ID 284577_1 is duplicated\n",
      "Tax ID 119000_0 is duplicated\n",
      "Tax ID 1121943_0 is duplicated\n",
      "Tax ID 2032625_0 is duplicated\n",
      "Tax ID 1666906_0 is duplicated\n",
      "Tax ID 416873_0 is duplicated\n",
      "Tax ID 1684425_0 is duplicated\n",
      "Tax ID 1975722_0 is duplicated\n",
      "Tax ID 115561_0 is duplicated\n",
      "Tax ID 272774_0 is duplicated\n",
      "Tax ID 2042704_0 is duplicated\n",
      "Tax ID 1962264_0 is duplicated\n",
      "Tax ID 232346_0 is duplicated\n",
      "Tax ID 1118153_0 is duplicated\n",
      "Tax ID 44935_0 is duplicated\n",
      "Tax ID 447567_0 is duplicated\n",
      "Tax ID 213554_1 is duplicated\n",
      "Tax ID 1971364_0 is duplicated\n",
      "Tax ID 999141_0 is duplicated\n",
      "Tax ID 29570_1 is duplicated\n",
      "Tax ID 29570_0 is duplicated\n",
      "Tax ID 2585770_0 is duplicated\n",
      "Tax ID 115553_0 is duplicated\n",
      "Tax ID 31910_0 is duplicated\n",
      "Tax ID 213554_0 is duplicated\n",
      "Tax ID 1072583_0 is duplicated\n",
      "Tax ID 2058313_0 is duplicated\n",
      "Tax ID 232346_1 is duplicated\n",
      "Tax ID 1123773_0 is duplicated\n",
      "Tax ID 1610576_0 is duplicated\n",
      "Tax ID 1805819_0 is duplicated\n",
      "Tax ID 1121940_0 is duplicated\n",
      "Tax ID 44933_0 is duplicated\n",
      "Tax ID 2483111_0 is duplicated\n",
      "Tax ID 42566_0 is duplicated\n",
      "Tax ID 2033802_0 is duplicated\n",
      "Tax ID 447425_0 is duplicated\n",
      "Tax ID 475662_0 is duplicated\n",
      "Tax ID 2508710_0 is duplicated\n",
      "Tax ID 2546095_0 is duplicated\n",
      "Tax ID 419597_0 is duplicated\n",
      "Tax ID 147248_0 is duplicated\n",
      "Tax ID 229007_1 is duplicated\n",
      "Tax ID 42565_0 is duplicated\n",
      "Tax ID 1897729_0 is duplicated\n",
      "Tax ID 416872_0 is duplicated\n",
      "Tax ID 2587844_0 is duplicated\n",
      "Tax ID 657387_0 is duplicated\n",
      "Tax ID 1949085_0 is duplicated\n",
      "Tax ID 404433_0 is duplicated\n",
      "Tax ID 1928762_0 is duplicated\n",
      "Tax ID 1267424_0 is duplicated\n",
      "Tax ID 184065_0 is duplicated\n",
      "Tax ID 1949082_0 is duplicated\n",
      "Tax ID 1960125_0 is duplicated\n",
      "Tax ID 1698521_0 is duplicated\n",
      "Tax ID 1121862_0 is duplicated\n",
      "Tax ID 1333660_0 is duplicated\n",
      "Tax ID 570277_0 is duplicated\n",
      "Tax ID 1914300_0 is duplicated\n",
      "Tax ID 2072845_0 is duplicated\n",
      "Tax ID 1260251_0 is duplicated\n",
      "Tax ID 1838287_0 is duplicated\n",
      "Tax ID 1335757_0 is duplicated\n",
      "Tax ID 531306_0 is duplicated\n",
      "Tax ID 1052_0 is duplicated\n",
      "Tax ID 349124_0 is duplicated\n",
      "Tax ID 1781069_0 is duplicated\n",
      "Tax ID 2571147_0 is duplicated\n",
      "Tax ID 1123516_0 is duplicated\n",
      "Tax ID 1918949_0 is duplicated\n",
      "Tax ID 1123397_0 is duplicated\n",
      "Tax ID 381431_0 is duplicated\n",
      "Tax ID 489703_0 is duplicated\n",
      "Tax ID 2029166_0 is duplicated\n",
      "Tax ID 1469245_0 is duplicated\n",
      "Tax ID 40754_0 is duplicated\n",
      "Tax ID 1675686_0 is duplicated\n",
      "Tax ID 1685380_0 is duplicated\n",
      "Tax ID 1123517_0 is duplicated\n",
      "Tax ID 2065820_0 is duplicated\n",
      "Tax ID 1168067_0 is duplicated\n",
      "Tax ID 2267253_0 is duplicated\n",
      "Tax ID 1705394_0 is duplicated\n",
      "Tax ID 1705394_0 is duplicated\n",
      "Tax ID 2080658_0 is duplicated\n",
      "Tax ID 491947_0 is duplicated\n",
      "Tax ID 717774_0 is duplicated\n",
      "Tax ID 314277_0 is duplicated\n",
      "Tax ID 579484_0 is duplicated\n",
      "Tax ID 295068_0 is duplicated\n",
      "Tax ID 1137284_0 is duplicated\n",
      "Tax ID 491954_0 is duplicated\n",
      "Tax ID 2530201_0 is duplicated\n",
      "Tax ID 1611323_0 is duplicated\n",
      "Tax ID 85925_0 is duplicated\n",
      "Tax ID 187272_0 is duplicated\n",
      "Tax ID 2182432_0 is duplicated\n",
      "Tax ID 1121016_0 is duplicated\n",
      "Tax ID 1972068_0 is duplicated\n",
      "Tax ID 585455_1 is duplicated\n",
      "Tax ID 585455_0 is duplicated\n",
      "Tax ID 415747_0 is duplicated\n",
      "Tax ID 947516_0 is duplicated\n",
      "Tax ID 1249627_0 is duplicated\n",
      "Tax ID 765913_0 is duplicated\n",
      "Tax ID 765912_0 is duplicated\n",
      "Tax ID 316276_0 is duplicated\n",
      "Tax ID 231048_0 is duplicated\n",
      "Tax ID 572477_0 is duplicated\n",
      "Tax ID 765911_0 is duplicated\n",
      "Tax ID 127458_0 is duplicated\n",
      "Tax ID 768671_0 is duplicated\n",
      "Tax ID 69360_0 is duplicated\n",
      "Tax ID 610332_0 is duplicated\n",
      "Tax ID 1058_0 is duplicated\n",
      "Tax ID 2340_0 is duplicated\n",
      "Tax ID 1076588_0 is duplicated\n",
      "Tax ID 54398_0 is duplicated\n",
      "Tax ID 191960_0 is duplicated\n",
      "Tax ID 1543721_0 is duplicated\n",
      "Tax ID 1111735_0 is duplicated\n",
      "Tax ID 525918_0 is duplicated\n",
      "Tax ID 713585_0 is duplicated\n",
      "Tax ID 1255043_0 is duplicated\n",
      "Tax ID 867345_0 is duplicated\n",
      "Tax ID 519989_0 is duplicated\n",
      "Tax ID 195064_0 is duplicated\n",
      "Tax ID 1396821_0 is duplicated\n",
      "Tax ID 421628_0 is duplicated\n",
      "Tax ID 1442136_0 is duplicated\n",
      "Tax ID 547122_0 is duplicated\n",
      "Tax ID 765914_0 is duplicated\n",
      "Tax ID 1033854_0 is duplicated\n",
      "Tax ID 396588_0 is duplicated\n",
      "Tax ID 108003_0 is duplicated\n",
      "Tax ID 713586_0 is duplicated\n",
      "Tax ID 1122603_0 is duplicated\n",
      "Tax ID 1884353_0 is duplicated\n",
      "Tax ID 1919059_0 is duplicated\n",
      "Tax ID 2516328_0 is duplicated\n",
      "Tax ID 1317117_0 is duplicated\n",
      "Tax ID 1765967_0 is duplicated\n",
      "Tax ID 406100_0 is duplicated\n",
      "Tax ID 92487_0 is duplicated\n",
      "Tax ID 1400861_0 is duplicated\n",
      "Tax ID 1400863_0 is duplicated\n",
      "Tax ID 545276_0 is duplicated\n",
      "Tax ID 252474_0 is duplicated\n",
      "Tax ID 748658_0 is duplicated\n",
      "Tax ID 545264_0 is duplicated\n",
      "Tax ID 1158148_0 is duplicated\n",
      "Tax ID 1158150_0 is duplicated\n",
      "Tax ID 1158152_0 is duplicated\n",
      "Tax ID 1158151_0 is duplicated\n",
      "Tax ID 396595_0 is duplicated\n",
      "Tax ID 1158762_0 is duplicated\n",
      "Tax ID 1158146_0 is duplicated\n",
      "Tax ID 106634_0 is duplicated\n",
      "Tax ID 1158165_0 is duplicated\n",
      "Tax ID 1158172_0 is duplicated\n",
      "Tax ID 748654_0 is duplicated\n",
      "Tax ID 1266908_0 is duplicated\n",
      "Tax ID 1266914_0 is duplicated\n",
      "Tax ID 1123515_0 is duplicated\n",
      "Tax ID 2580412_0 is duplicated\n",
      "Tax ID 1123518_0 is duplicated\n",
      "Tax ID 1123513_0 is duplicated\n",
      "Tax ID 1803865_0 is duplicated\n",
      "Tax ID 1775636_0 is duplicated\n",
      "Tax ID 39765_0 is duplicated\n",
      "Tax ID 317025_0 is duplicated\n",
      "Tax ID 1620215_0 is duplicated\n",
      "Tax ID 381308_0 is duplicated\n",
      "Tax ID 1750597_0 is duplicated\n",
      "Tax ID 1818881_0 is duplicated\n",
      "Tax ID 1655433_0 is duplicated\n",
      "Tax ID 765910_0 is duplicated\n",
      "Tax ID 1166950_0 is duplicated\n",
      "Tax ID 765909_0 is duplicated\n",
      "Tax ID 1431688_0 is duplicated\n",
      "Tax ID 424902_0 is duplicated\n",
      "Tax ID 631362_0 is duplicated\n",
      "Tax ID 351052_0 is duplicated\n",
      "Tax ID 642797_0 is duplicated\n",
      "Tax ID 1899563_0 is duplicated\n",
      "Tax ID 1123033_0 is duplicated\n",
      "Tax ID 1002339_0 is duplicated\n",
      "Tax ID 1945520_0 is duplicated\n",
      "Tax ID 1123034_0 is duplicated\n",
      "Tax ID 861445_0 is duplicated\n",
      "Tax ID 1975705_0 is duplicated\n",
      "Tax ID 2488563_0 is duplicated\n",
      "Tax ID 494416_0 is duplicated\n",
      "Tax ID 1849383_0 is duplicated\n",
      "Tax ID 1926478_0 is duplicated\n",
      "Tax ID 1982756_0 is duplicated\n",
      "Tax ID 198480_0 is duplicated\n",
      "Tax ID 1699622_0 is duplicated\n",
      "Tax ID 571800_0 is duplicated\n",
      "Tax ID 1699621_0 is duplicated\n",
      "Tax ID 2597355_0 is duplicated\n",
      "Tax ID 1028416_0 is duplicated\n",
      "Tax ID 470453_0 is duplicated\n",
      "Tax ID 1775913_0 is duplicated\n",
      "Tax ID 332522_0 is duplicated\n",
      "Tax ID 86177_0 is duplicated\n",
      "Tax ID 86177_1 is duplicated\n",
      "Tax ID 2032626_0 is duplicated\n",
      "Tax ID 2597518_0 is duplicated\n",
      "Tax ID 141451_0 is duplicated\n",
      "Tax ID 1982043_0 is duplicated\n",
      "Tax ID 488535_0 is duplicated\n",
      "Tax ID 488533_0 is duplicated\n",
      "Tax ID 2603215_0 is duplicated\n",
      "Tax ID 626887_0 is duplicated\n",
      "Tax ID 2201898_0 is duplicated\n",
      "Tax ID 2528979_0 is duplicated\n",
      "Tax ID 2484250_0 is duplicated\n",
      "Tax ID 650891_0 is duplicated\n",
      "Tax ID 430453_0 is duplicated\n",
      "Tax ID 1569211_0 is duplicated\n",
      "Tax ID 2597703_0 is duplicated\n",
      "Tax ID 1479237_0 is duplicated\n",
      "Tax ID 2576384_0 is duplicated\n",
      "Tax ID 1928331_0 is duplicated\n",
      "Tax ID 1928859_0 is duplicated\n",
      "Tax ID 1874317_0 is duplicated\n",
      "Tax ID 1671721_0 is duplicated\n",
      "Tax ID 2055143_0 is duplicated\n",
      "Tax ID 1761795_0 is duplicated\n",
      "Tax ID 1033846_0 is duplicated\n",
      "Tax ID 2592624_0 is duplicated\n",
      "Tax ID 1849167_0 is duplicated\n",
      "Tax ID 432303_0 is duplicated\n",
      "Tax ID 2304594_0 is duplicated\n",
      "Tax ID 2039467_0 is duplicated\n",
      "Tax ID 1925763_0 is duplicated\n",
      "Tax ID 564117_0 is duplicated\n",
      "Tax ID 277961_0 is duplicated\n",
      "Tax ID 1494078_0 is duplicated\n",
      "Tax ID 1288826_0 is duplicated\n",
      "Tax ID 379482_0 is duplicated\n",
      "Tax ID 379482_1 is duplicated\n",
      "Tax ID 930118_0 is duplicated\n",
      "Tax ID 1137280_0 is duplicated\n",
      "Tax ID 1122197_0 is duplicated\n",
      "Tax ID 351348_0 is duplicated\n",
      "Tax ID 1323740_0 is duplicated\n",
      "Tax ID 1305731_0 is duplicated\n",
      "Tax ID 247634_0 is duplicated\n",
      "Tax ID 2026743_5 is duplicated\n",
      "Tax ID 2026743_11 is duplicated\n",
      "Tax ID 2026743_4 is duplicated\n",
      "Tax ID 2026743_10 is duplicated\n",
      "Tax ID 2026743_9 is duplicated\n",
      "Tax ID 2026743_2 is duplicated\n",
      "Tax ID 1434821_0 is duplicated\n",
      "Tax ID 141451_1 is duplicated\n",
      "Tax ID 64971_0 is duplicated\n",
      "Tax ID 1122212_0 is duplicated\n",
      "Tax ID 1122211_0 is duplicated\n",
      "Tax ID 1122209_0 is duplicated\n",
      "Tax ID 1982602_0 is duplicated\n",
      "Tax ID 1122252_0 is duplicated\n",
      "Tax ID 83406_0 is duplicated\n",
      "Tax ID 207949_0 is duplicated\n",
      "Tax ID 1730894_0 is duplicated\n",
      "Tax ID 1121103_0 is duplicated\n",
      "Tax ID 2490544_0 is duplicated\n",
      "Tax ID 355243_0 is duplicated\n",
      "Tax ID 1278309_0 is duplicated\n",
      "Tax ID 207954_0 is duplicated\n",
      "Tax ID 1806670_0 is duplicated\n",
      "Tax ID 1806669_0 is duplicated\n",
      "Tax ID 1902503_0 is duplicated\n",
      "Tax ID 1792290_0 is duplicated\n",
      "Tax ID 2529379_0 is duplicated\n",
      "Tax ID 28254_0 is duplicated\n",
      "Tax ID 491952_0 is duplicated\n",
      "Tax ID 491951_0 is duplicated\n",
      "Tax ID 1561203_0 is duplicated\n",
      "Tax ID 491950_0 is duplicated\n",
      "Tax ID 2071621_0 is duplicated\n",
      "Tax ID 1122207_0 is duplicated\n",
      "Tax ID 383750_0 is duplicated\n",
      "Tax ID 2202418_0 is duplicated\n",
      "Tax ID 491948_0 is duplicated\n",
      "Tax ID 1122206_0 is duplicated\n",
      "Tax ID 400668_0 is duplicated\n",
      "Tax ID 1208321_0 is duplicated\n",
      "Tax ID 178399_0 is duplicated\n",
      "Tax ID 2029982_3 is duplicated\n",
      "Tax ID 2029982_0 is duplicated\n",
      "Tax ID 1848729_0 is duplicated\n",
      "Tax ID 1194167_0 is duplicated\n",
      "Tax ID 1177156_0 is duplicated\n",
      "Tax ID 930169_0 is duplicated\n",
      "Tax ID 1817793_0 is duplicated\n",
      "Tax ID 2283196_0 is duplicated\n",
      "Tax ID 64969_0 is duplicated\n",
      "Tax ID 1122616_0 is duplicated\n",
      "Tax ID 1895005_0 is duplicated\n",
      "Tax ID 377629_0 is duplicated\n",
      "Tax ID 1056820_0 is duplicated\n",
      "Tax ID 1403320_0 is duplicated\n",
      "Tax ID 1123399_0 is duplicated\n",
      "Tax ID 2506149_0 is duplicated\n",
      "Tax ID 1897630_0 is duplicated\n",
      "Tax ID 2561897_0 is duplicated\n",
      "Tax ID 1572645_0 is duplicated\n",
      "Tax ID 1031538_0 is duplicated\n",
      "Tax ID 1122599_0 is duplicated\n",
      "Tax ID 1045558_0 is duplicated\n",
      "Tax ID 619304_0 is duplicated\n",
      "Tax ID 452629_0 is duplicated\n",
      "Tax ID 1122200_0 is duplicated\n",
      "Tax ID 1122202_0 is duplicated\n",
      "Tax ID 1821621_0 is duplicated\n",
      "Tax ID 1714300_0 is duplicated\n",
      "Tax ID 1708751_0 is duplicated\n",
      "Tax ID 1571224_0 is duplicated\n",
      "Tax ID 267850_0 is duplicated\n",
      "Tax ID 1628392_0 is duplicated\n",
      "Tax ID 349521_0 is duplicated\n",
      "Tax ID 1121935_0 is duplicated\n",
      "Tax ID 1926491_0 is duplicated\n",
      "Tax ID 1822242_0 is duplicated\n",
      "Tax ID 1177159_0 is duplicated\n",
      "Tax ID 857227_0 is duplicated\n",
      "Tax ID 254161_0 is duplicated\n",
      "Tax ID 1708783_0 is duplicated\n",
      "Tax ID 857252_0 is duplicated\n",
      "Tax ID 1434072_0 is duplicated\n",
      "Tax ID 2078689_0 is duplicated\n",
      "Tax ID 472181_0 is duplicated\n",
      "Tax ID 2487347_0 is duplicated\n",
      "Tax ID 1981513_0 is duplicated\n",
      "Tax ID 2586906_0 is duplicated\n",
      "Tax ID 487184_0 is duplicated\n",
      "Tax ID 1323744_0 is duplicated\n",
      "Tax ID 1323663_0 is duplicated\n",
      "Tax ID 553151_0 is duplicated\n",
      "Tax ID 2268758_0 is duplicated\n"
     ]
    }
   ],
   "source": [
    "filtered_files = [f for f in os.listdir() if re.match(r'[A-D]_filtered\\.fasta', f)]\n",
    "# check duplicated tax IDs\n",
    "\n",
    "for file in filtered_files:\n",
    "    # read the file with the Bio.SeqIO module\n",
    "    headers = []\n",
    "    tax_ids = []\n",
    "    seqs = []\n",
    "    for seq_record in SeqIO.parse(file, \"fasta\"):\n",
    "        header = '>' + str(seq_record.description)\n",
    "        headers.append(header)\n",
    "        seq = str(seq_record.seq)\n",
    "        seqs.append(seq)\n",
    "        # print(header)\n",
    "        tax_id = header.split(\"organism taxid:\")[1].split(\",\")[0][2:-1]\n",
    "        # print(tax_id)\n",
    "        if tax_id not in tax_ids:\n",
    "            tax_ids.append(tax_id)\n",
    "        else:\n",
    "            print(\"Tax ID {} is duplicated\".format(tax_id))        "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "DL",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}