Labeled LDA.ipynb

{
 "metadata": {
  "name": "Labeled LDA"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "source: http://shuyo.wordpress.com/2013/07/24/python-implementation-of-labeled-lda-ramage-emnlp2009/\n",
      "\n",
      "llda.py is LLDA estimation for arbitrary test. Its input file consits on each line as a document.\n",
      "Each document can be given label(s) to put in brackets at the head of the line, like the following:\n",
      "\n",
      "[label1,label2,...]some text\n",
      "To give documents without labels, it works like semi-supervised.\n",
      "\n",
      "llda_nltk.py is a sample with llda.py. It estimates LLDA model with 100 sampled documents from Reuters corpus in NLTK and outputs top 20 words for each label."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# llda.py\n",
      "# Labeled Latent Dirichlet Allocation\n",
      "# This code is available under the MIT License.\n",
      "# (c)2010 Nakatani Shuyo / Cybozu Labs Inc.\n",
      "# refer to Ramage+, Labeled LDA: A supervised topic model for credit attribution in multi-labeled corpora(EMNLP2009)\n",
      "\n",
      "from optparse import OptionParser\n",
      "import sys, re, numpy\n",
      "\n",
      "def load_corpus(filename):\n",
      "    corpus = []\n",
      "    labels = []\n",
      "    labelmap = dict()\n",
      "    f = open(filename, 'r')\n",
      "    for line in f:\n",
      "        mt = re.match(r'\\[(.+?)\\](.+)', line)\n",
      "        if mt:\n",
      "            label = mt.group(1).split(',')\n",
      "            for x in label: labelmap[x] = 1\n",
      "            line = mt.group(2)\n",
      "        else:\n",
      "            label = None\n",
      "        doc = re.findall(r'\\w+(?:\\'\\w+)?',line.lower())\n",
      "        if len(doc)>0:\n",
      "            corpus.append(doc)\n",
      "            labels.append(label)\n",
      "    f.close()\n",
      "    return labelmap.keys(), corpus, labels\n",
      "\n",
      "class LLDA:\n",
      "    def __init__(self, K, alpha, beta):\n",
      "        #self.K = K\n",
      "        self.alpha = alpha\n",
      "        self.beta = beta\n",
      "\n",
      "    def term_to_id(self, term):\n",
      "        if term not in self.vocas_id:\n",
      "            voca_id = len(self.vocas)\n",
      "            self.vocas_id[term] = voca_id\n",
      "            self.vocas.append(term)\n",
      "        else:\n",
      "            voca_id = self.vocas_id[term]\n",
      "        return voca_id\n",
      "\n",
      "    def complement_label(self, label):\n",
      "        if not label: return numpy.ones(len(self.labelmap))\n",
      "        vec = numpy.zeros(len(self.labelmap))\n",
      "        vec[0] = 1.0\n",
      "        for x in label: vec[self.labelmap[x]] = 1.0\n",
      "        return vec\n",
      "\n",
      "    def set_corpus(self, labelset, corpus, labels):\n",
      "        labelset.insert(0, \"common\")\n",
      "        self.labelmap = dict(zip(labelset, range(len(labelset))))\n",
      "        self.K = len(self.labelmap)\n",
      "\n",
      "        self.vocas = []\n",
      "        self.vocas_id = dict()\n",
      "        self.labels = numpy.array([self.complement_label(label) for label in labels])\n",
      "        self.docs = [[self.term_to_id(term) for term in doc] for doc in corpus]\n",
      "\n",
      "        M = len(corpus)\n",
      "        V = len(self.vocas)\n",
      "\n",
      "        self.z_m_n = []\n",
      "        self.n_m_z = numpy.zeros((M, self.K), dtype=int)\n",
      "        self.n_z_t = numpy.zeros((self.K, V), dtype=int)\n",
      "        self.n_z = numpy.zeros(self.K, dtype=int)\n",
      "\n",
      "        for m, doc, label in zip(range(M), self.docs, self.labels):\n",
      "            N_m = len(doc)\n",
      "            #z_n = [label[x] for x in numpy.random.randint(len(label), size=N_m)]\n",
      "            z_n = [numpy.random.multinomial(1, label / label.sum()).argmax() for x in range(N_m)]\n",
      "            self.z_m_n.append(z_n)\n",
      "            for t, z in zip(doc, z_n):\n",
      "                self.n_m_z[m, z] += 1\n",
      "                self.n_z_t[z, t] += 1\n",
      "                self.n_z[z] += 1\n",
      "\n",
      "    def inference(self):\n",
      "        V = len(self.vocas)\n",
      "        for m, doc, label in zip(range(len(self.docs)), self.docs, self.labels):\n",
      "            for n in range(len(doc)):\n",
      "                t = doc[n]\n",
      "                z = self.z_m_n[m][n]\n",
      "                self.n_m_z[m, z] -= 1\n",
      "                self.n_z_t[z, t] -= 1\n",
      "                self.n_z[z] -= 1\n",
      "\n",
      "                denom_a = self.n_m_z[m].sum() + self.K * self.alpha\n",
      "                denom_b = self.n_z_t.sum(axis=1) + V * self.beta\n",
      "                p_z = label * (self.n_z_t[:, t] + self.beta) / denom_b * (self.n_m_z[m] + self.alpha) / denom_a\n",
      "                new_z = numpy.random.multinomial(1, p_z / p_z.sum()).argmax()\n",
      "\n",
      "                self.z_m_n[m][n] = new_z\n",
      "                self.n_m_z[m, new_z] += 1\n",
      "                self.n_z_t[new_z, t] += 1\n",
      "                self.n_z[new_z] += 1\n",
      "\n",
      "    def phi(self):\n",
      "        V = len(self.vocas)\n",
      "        return (self.n_z_t + self.beta) / (self.n_z[:, numpy.newaxis] + V * self.beta)\n",
      "\n",
      "    def theta(self):\n",
      "        \"\"\"document-topic distribution\"\"\"\n",
      "        n_alpha = self.n_m_z + self.labels * self.alpha\n",
      "        return n_alpha / n_alpha.sum(axis=1)[:, numpy.newaxis]\n",
      "\n",
      "    def perplexity(self, docs=None):\n",
      "        if docs == None: docs = self.docs\n",
      "        phi = self.phi()\n",
      "        thetas = self.theta()\n",
      "\n",
      "        log_per = N = 0\n",
      "        for doc, theta in zip(docs, thetas):\n",
      "            for w in doc:\n",
      "                log_per -= numpy.log(numpy.inner(phi[:,w], theta))\n",
      "            N += len(doc)\n",
      "        return numpy.exp(log_per / N)\n",
      "\n",
      "def main():\n",
      "    parser = OptionParser()\n",
      "    parser.add_option(\"-f\", dest=\"filename\", help=\"corpus filename\")\n",
      "    parser.add_option(\"--alpha\", dest=\"alpha\", type=\"float\", help=\"parameter alpha\", default=0.001)\n",
      "    parser.add_option(\"--beta\", dest=\"beta\", type=\"float\", help=\"parameter beta\", default=0.001)\n",
      "    parser.add_option(\"-k\", dest=\"K\", type=\"int\", help=\"number of topics\", default=20)\n",
      "    parser.add_option(\"-i\", dest=\"iteration\", type=\"int\", help=\"iteration count\", default=100)\n",
      "    (options, args) = parser.parse_args()\n",
      "    if not options.filename: parser.error(\"need corpus filename(-f)\")\n",
      "\n",
      "    labelset, corpus, labels = load_corpus(options.filename)\n",
      "\n",
      "    llda = LLDA(options.K, options.alpha, options.beta)\n",
      "    llda.set_corpus(labelset, corpus, labels)\n",
      "\n",
      "    for i in range(options.iteration):\n",
      "        sys.stderr.write(\"-- %d \" % (i + 1))\n",
      "        llda.inference()\n",
      "    #print llda.z_m_n\n",
      "\n",
      "    phi = llda.phi()\n",
      "    for v, voca in enumerate(llda.vocas):\n",
      "        #print ','.join([voca]+[str(x) for x in llda.n_z_t[:,v]])\n",
      "        print ','.join([voca]+[str(x) for x in phi[:,v]])\n",
      "\n",
      "#if __name__ == \"__main__\":\n",
      "#    main()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# llda_nltk.py\n",
      "# Labeled LDA using nltk.corpus.reuters as dataset\n",
      "# This code is available under the MIT License.\n",
      "# (c)2013 Nakatani Shuyo / Cybozu Labs Inc.\n",
      "\n",
      "import sys, string, random, numpy\n",
      "from nltk.corpus import reuters\n",
      "#from llda import LLDA\n",
      "from optparse import OptionParser\n",
      "\n",
      "#parser = OptionParser()\n",
      "#parser.add_option(\"--alpha\", dest=\"alpha\", type=\"float\", help=\"parameter alpha\", default=0.001)\n",
      "#parser.add_option(\"--beta\", dest=\"beta\", type=\"float\", help=\"parameter beta\", default=0.001)\n",
      "#parser.add_option(\"-k\", dest=\"K\", type=\"int\", help=\"number of topics\", default=50)\n",
      "#parser.add_option(\"-i\", dest=\"iteration\", type=\"int\", help=\"iteration count\", default=100)\n",
      "#parser.add_option(\"-s\", dest=\"seed\", type=\"int\", help=\"random seed\", default=None)\n",
      "#parser.add_option(\"-n\", dest=\"samplesize\", type=\"int\", help=\"dataset sample size\", default=100)\n",
      "#(options, args) = parser.parse_args()\n",
      "\n",
      "#random.seed(options.seed)\n",
      "random.seed(None)\n",
      "#numpy.random.seed(options.seed)\n",
      "numpy.random.seed(None)\n",
      "\n",
      "#idlist = random.sample(reuters.fileids(), options.samplesize)\n",
      "idlist = random.sample(reuters.fileids(), 100)\n",
      "\n",
      "labels = []\n",
      "corpus = []\n",
      "for id in idlist:\n",
      "    labels.append(reuters.categories(id))\n",
      "    corpus.append([x.lower() for x in reuters.words(id) if x[0] in string.ascii_letters])\n",
      "    reuters.words(id).close()\n",
      "labelset = list(set(reduce(list.__add__, labels)))\n",
      "\n",
      "\n",
      "#llda = LLDA(options.K, options.alpha, options.beta)\n",
      "llda = LLDA(50, 0.001, 0.001)\n",
      "llda.set_corpus(labelset, corpus, labels)\n",
      "\n",
      "#print \"M=%d, V=%d, L=%d, K=%d\" % (len(corpus), len(llda.vocas), len(labelset), options.K)\n",
      "print \"M=%d, V=%d, L=%d, K=%d\" % (len(corpus), len(llda.vocas), len(labelset), 50)\n",
      "\n",
      "#for i in range(options.iteration):\n",
      "for i in range(100):\n",
      "    sys.stderr.write(\"-- %d : %.4f\\n\" % (i, llda.perplexity()))\n",
      "    llda.inference()\n",
      "print \"perplexity : %.4f\" % llda.perplexity()\n",
      "\n",
      "phi = llda.phi()\n",
      "for k, label in enumerate(labelset):\n",
      "    print \"\\n-- label %d : %s\" % (k, label)\n",
      "    for w in numpy.argsort(-phi[k])[:20]:\n",
      "        print \"%s: %.4f\" % (llda.vocas[w], phi[k,w])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 0 : 278.5829\n",
        "-- 1 : 240.3652\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 2 : 237.9194\n",
        "-- 3 : 235.9442\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 4 : 237.0411\n",
        "-- 5 : 236.5012\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 6 : 238.2019\n",
        "-- 7 : 237.9127\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 8 : 237.7013\n",
        "-- 9 : 237.6881\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 10 : 238.2866\n",
        "-- 11 : 238.1215\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 12 : 238.0206\n",
        "-- 13 : 236.8641\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 14 : 237.5517\n",
        "-- 15 : 236.9801\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 16 : 236.8041\n",
        "-- 17 : 236.7620\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 18 : 237.9574\n",
        "-- 19 : 236.3425\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 20 : 236.9332\n",
        "-- 21 : 237.9855\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 22 : 238.0776\n",
        "-- 23 : 237.6373\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 24 : 236.7682\n",
        "-- 25 : 236.3494\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 26 : 236.6016\n",
        "-- 27 : 235.7818\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 28 : 236.5745\n",
        "-- 29 : 237.2096\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 30 : 235.3374\n",
        "-- 31 : 236.4600\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 32 : 234.8948\n",
        "-- 33 : 234.6708\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 34 : 234.9684\n",
        "-- 35 : 234.1949\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 36 : 235.5357\n",
        "-- 37 : 236.0604\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 38 : 235.6825\n",
        "-- 39 : 235.2438\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 40 : 236.2116\n",
        "-- 41 : 236.6391\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 42 : 236.0174\n",
        "-- 43 : 235.3376\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 44 : 235.6781\n",
        "-- 45 : 235.0375\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 46 : 235.0071\n",
        "-- 47 : 235.0661\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 48 : 234.3484\n",
        "-- 49 : 235.0647\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 50 : 236.0557\n",
        "-- 51 : 234.2459\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 52 : 233.8836\n",
        "-- 53 : 233.3675\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 54 : 233.6783\n",
        "-- 55 : 233.6873\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 56 : 234.1758\n",
        "-- 57 : 233.7966\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 58 : 234.3733\n",
        "-- 59 : 233.5476\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 60 : 233.8270\n",
        "-- 61 : 232.3102\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 62 : 232.3330\n",
        "-- 63 : 233.2433\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 64 : 232.6177\n",
        "-- 65 : 232.8877\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 66 : 232.4790\n",
        "-- 67 : 232.6285\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 68 : 232.4641\n",
        "-- 69 : 232.3988\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 70 : 231.8806\n",
        "-- 71 : 232.3823\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 72 : 231.4466\n",
        "-- 73 : 232.9846\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 74 : 230.9360\n",
        "-- 75 : 230.9959\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 76 : 230.3905\n",
        "-- 77 : 230.6943\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 78 : 230.6950\n",
        "-- 79 : 230.4603\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 80 : 231.1901\n",
        "-- 81 : 231.1665\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 82 : 230.9326\n",
        "-- 83 : 230.9670\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 84 : 232.2537\n",
        "-- 85 : 231.3388\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 86 : 231.1295\n",
        "-- 87 : 230.3136\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 88 : 229.8887\n",
        "-- 89 : 230.9199\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 90 : 230.0063\n",
        "-- 91 : 230.2872\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 92 : 231.6411\n",
        "-- 93 : 230.7486\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 94 : 230.5708\n",
        "-- 95 : 230.9351\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 96 : 230.7536\n",
        "-- 97 : 230.5225\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "-- 98 : 230.4530\n",
        "-- 99 : 229.9730\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "M=100, V=2609, L=33, K=50\n",
        "perplexity : 230.9428"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "\n",
        "-- label 0 : common\n",
        "the: 0.0901\n",
        "to: 0.0446\n",
        "of: 0.0415\n",
        "a: 0.0348\n",
        "said: 0.0325\n",
        "in: 0.0324\n",
        "and: 0.0302\n",
        "s: 0.0178\n",
        "for: 0.0172\n",
        "that: 0.0123\n",
        "it: 0.0110\n",
        "at: 0.0106\n",
        "by: 0.0105\n",
        "is: 0.0103\n",
        "be: 0.0096\n",
        "pct: 0.0095\n",
        "on: 0.0095\n",
        "he: 0.0092\n",
        "its: 0.0089\n",
        "not: 0.0081\n",
        "\n",
        "-- label 1 : copper\n",
        "copper: 0.0958\n",
        "meeting: 0.0479\n",
        "study: 0.0479\n",
        "world: 0.0479\n",
        "objectives: 0.0479\n",
        "body: 0.0320\n",
        "functions: 0.0320\n",
        "aimed: 0.0320\n",
        "unctad: 0.0320\n",
        "held: 0.0320\n",
        "demand: 0.0160\n",
        "conference: 0.0160\n",
        "cleared: 0.0160\n",
        "negotiating: 0.0160\n",
        "session: 0.0160\n",
        "proposed: 0.0160\n",
        "depressed: 0.0160\n",
        "include: 0.0160\n",
        "stabilise: 0.0160\n",
        "statistics: 0.0160\n",
        "\n",
        "-- label 2 : livestock\n",
        "japan: 0.1613\n",
        "tonnes: 0.1613\n",
        "quota: 0.1075\n",
        "half: 0.1075\n",
        "beef: 0.1075\n",
        "increasing: 0.0538\n",
        "projected: 0.0538\n",
        "ministry: 0.0538\n",
        "australia: 0.0538\n",
        "bryson: 0.0001\n",
        "pays: 0.0001\n",
        "cenergy: 0.0001\n",
        "gas: 0.0001\n",
        "paid: 0.0001\n",
        "expand: 0.0001\n",
        "investments: 0.0001\n",
        "dixons: 0.0001\n",
        "believes: 0.0001\n",
        "opportunity: 0.0001\n",
        "provides: 0.0001\n",
        "\n",
        "-- label 3 : gold\n",
        "plant: 0.0621\n",
        "gold: 0.0518\n",
        "ore: 0.0414\n",
        "nerco: 0.0414\n",
        "process: 0.0414\n",
        "alligator: 0.0311\n",
        "lt: 0.0311\n",
        "bp: 0.0311\n",
        "amselco: 0.0311\n",
        "mine: 0.0311\n",
        "ridge: 0.0311\n",
        "will: 0.0311\n",
        "unit: 0.0311\n",
        "minerals: 0.0207\n",
        "reserves: 0.0207\n",
        "leaching: 0.0207\n",
        "inc: 0.0207\n",
        "ounces: 0.0207\n",
        "carbon: 0.0207\n",
        "recover: 0.0207\n",
        "\n",
        "-- label 4 : money-fx\n",
        "stoltenberg: 0.0570\n",
        "germany: 0.0326\n",
        "west: 0.0285\n",
        "in: 0.0285\n",
        "paris: 0.0244\n",
        "to: 0.0244\n",
        "baker: 0.0244\n",
        "policy: 0.0204\n",
        "monetary: 0.0163\n",
        "stg: 0.0163\n",
        "economic: 0.0163\n",
        "accord: 0.0163\n",
        "nominal: 0.0122\n",
        "currency: 0.0122\n",
        "imbalances: 0.0122\n",
        "fell: 0.0122\n",
        "sustainable: 0.0122\n",
        "within: 0.0122\n",
        "on: 0.0122\n",
        "government: 0.0122\n",
        "\n",
        "-- label 5 : trade\n",
        "trade: 0.0880\n",
        "bill: 0.0474\n",
        "reagan: 0.0304\n",
        "house: 0.0271\n",
        "legislation: 0.0169\n",
        "subcommittee: 0.0169\n",
        "his: 0.0169\n",
        "unfair: 0.0135\n",
        "lawmakers: 0.0135\n",
        "democrat: 0.0135\n",
        "free: 0.0135\n",
        "industries: 0.0102\n",
        "agreements: 0.0102\n",
        "means: 0.0102\n",
        "think: 0.0102\n",
        "states: 0.0102\n",
        "gephardt: 0.0102\n",
        "relief: 0.0102\n",
        "rep: 0.0102\n",
        "enforcement: 0.0068\n",
        "\n",
        "-- label 6 : cocoa\n",
        "icco: 0.0813\n",
        "cocoa: 0.0813\n",
        "manager: 0.0813\n",
        "stock: 0.0813\n",
        "june: 0.0813\n",
        "official: 0.0813\n",
        "tonnes: 0.0813\n",
        "monday: 0.0813\n",
        "to: 0.0813\n",
        "buy: 0.0813\n",
        "buffer: 0.0813\n",
        "unsuccessfully: 0.0000\n",
        "bid: 0.0000\n",
        "dixons: 0.0000\n",
        "bryson: 0.0000\n",
        "pays: 0.0000\n",
        "cenergy: 0.0000\n",
        "paid: 0.0000\n",
        "cyclops: 0.0000\n",
        "exploration: 0.0000\n",
        "\n",
        "-- label 7 : zinc\n",
        "cominco: 0.1186\n",
        "locals: 0.0988\n",
        "three: 0.0791\n",
        "workers: 0.0791\n",
        "produced: 0.0593\n",
        "may: 0.0593\n",
        "lead: 0.0593\n",
        "office: 0.0395\n",
        "vancouver: 0.0395\n",
        "b: 0.0198\n",
        "representing: 0.0198\n",
        "classifications: 0.0198\n",
        "expired: 0.0198\n",
        "living: 0.0198\n",
        "reopening: 0.0198\n",
        "america: 0.0198\n",
        "along: 0.0198\n",
        "declare: 0.0198\n",
        "cost: 0.0198\n",
        "requested: 0.0198\n",
        "\n",
        "-- label 8 : ship\n",
        "gulf: 0.0460\n",
        "iran: 0.0460\n",
        "the: 0.0394\n",
        "missile: 0.0295\n",
        "attack: 0.0230\n",
        "hormuz: 0.0230\n",
        "shipping: 0.0230\n",
        "iranian: 0.0197\n",
        "arab: 0.0197\n",
        "tehran: 0.0197\n",
        "diplomats: 0.0164\n",
        "naval: 0.0164\n",
        "states: 0.0164\n",
        "source: 0.0164\n",
        "least: 0.0164\n",
        "silkworm: 0.0164\n",
        "strait: 0.0164\n",
        "fired: 0.0131\n",
        "test: 0.0131\n",
        "diplomat: 0.0131\n",
        "\n",
        "-- label 9 : alum\n",
        "billiton: 0.0718\n",
        "surinam: 0.0598\n",
        "lt: 0.0479\n",
        "losses: 0.0359\n",
        "bauxite: 0.0359\n",
        "mining: 0.0239\n",
        "dutch: 0.0239\n",
        "suralco: 0.0239\n",
        "urged: 0.0239\n",
        "heidweiler: 0.0239\n",
        "policies: 0.0239\n",
        "metals: 0.0120\n",
        "power: 0.0120\n",
        "plant: 0.0120\n",
        "afobakka: 0.0120\n",
        "moengo: 0.0120\n",
        "shell: 0.0120\n",
        "shut: 0.0120\n",
        "royal: 0.0120\n",
        "fully: 0.0120\n",
        "\n",
        "-- label 10 : soybean\n",
        "s: 0.0818\n",
        "u: 0.0701\n",
        "argentina: 0.0584\n",
        "differential: 0.0584\n",
        "retaliation: 0.0351\n",
        "argentine: 0.0351\n",
        "case: 0.0351\n",
        "representative: 0.0234\n",
        "drafting: 0.0234\n",
        "soybean: 0.0234\n",
        "nspa: 0.0234\n",
        "on: 0.0234\n",
        "options: 0.0234\n",
        "ernesto: 0.0117\n",
        "smith: 0.0117\n",
        "buenos: 0.0117\n",
        "aires: 0.0117\n",
        "levies: 0.0117\n",
        "processors: 0.0117\n",
        "michael: 0.0117\n",
        "\n",
        "-- label 11 : strategic-metal\n",
        "year: 0.1210\n",
        "smelter: 0.1210\n",
        "contract: 0.1008\n",
        "trail: 0.0807\n",
        "mine: 0.0605\n",
        "strike: 0.0605\n",
        "long: 0.0605\n",
        "miles: 0.0403\n",
        "east: 0.0403\n",
        "steelworkers: 0.0403\n",
        "means: 0.0202\n",
        "prepare: 0.0202\n",
        "issues: 0.0202\n",
        "sites: 0.0202\n",
        "rules: 0.0202\n",
        "bargaining: 0.0202\n",
        "ore: 0.0202\n",
        "keep: 0.0202\n",
        "rejected: 0.0202\n",
        "totalled: 0.0202\n",
        "\n",
        "-- label 12 : lead\n",
        "about: 0.0823\n",
        "union: 0.0823\n",
        "tons: 0.0617\n",
        "zinc: 0.0617\n",
        "technical: 0.0412\n",
        "maintenance: 0.0412\n",
        "representative: 0.0412\n",
        "kimberley: 0.0412\n",
        "contracts: 0.0412\n",
        "wage: 0.0412\n",
        "rejects: 0.0206\n",
        "honor: 0.0206\n",
        "revenues: 0.0206\n",
        "able: 0.0206\n",
        "pressed: 0.0206\n",
        "loosening: 0.0206\n",
        "majeure: 0.0206\n",
        "negotiated: 0.0206\n",
        "went: 0.0206\n",
        "job: 0.0206\n",
        "\n",
        "-- label 13 : sugar\n",
        "china: 0.0607\n",
        "sugar: 0.0557\n",
        "year: 0.0354\n",
        "tonnes: 0.0354\n",
        "mln: 0.0253\n",
        "fell: 0.0202\n",
        "stocks: 0.0202\n",
        "mills: 0.0202\n",
        "demand: 0.0202\n",
        "farmers: 0.0202\n",
        "diplomat: 0.0202\n",
        "press: 0.0152\n",
        "mu: 0.0152\n",
        "guangdong: 0.0152\n",
        "situation: 0.0152\n",
        "trader: 0.0152\n",
        "factor: 0.0101\n",
        "yuan: 0.0101\n",
        "figure: 0.0101\n",
        "provincial: 0.0101\n",
        "\n",
        "-- label 14 : rubber\n",
        "the: 0.0797\n",
        "new: 0.0493\n",
        "price: 0.0417\n",
        "will: 0.0379\n",
        "rubber: 0.0341\n",
        "tonnes: 0.0304\n",
        "pact: 0.0304\n",
        "stock: 0.0266\n",
        "buffer: 0.0266\n",
        "reference: 0.0228\n",
        "cts: 0.0190\n",
        "council: 0.0190\n",
        "inra: 0.0190\n",
        "present: 0.0152\n",
        "unless: 0.0152\n",
        "world: 0.0152\n",
        "decides: 0.0152\n",
        "sell: 0.0152\n",
        "may: 0.0152\n",
        "accord: 0.0152\n",
        "\n",
        "-- label 15 : veg-oil\n",
        "ec: 0.1056\n",
        "farm: 0.0603\n",
        "tax: 0.0528\n",
        "commission: 0.0453\n",
        "ministers: 0.0377\n",
        "andriessen: 0.0302\n",
        "community: 0.0302\n",
        "diplomats: 0.0302\n",
        "west: 0.0302\n",
        "proposed: 0.0226\n",
        "warned: 0.0226\n",
        "oils: 0.0226\n",
        "farmers: 0.0151\n",
        "opposition: 0.0151\n",
        "hole: 0.0151\n",
        "defended: 0.0151\n",
        "kohl: 0.0151\n",
        "budget: 0.0151\n",
        "subsidies: 0.0151\n",
        "rid: 0.0151\n",
        "\n",
        "-- label 16 : barley\n",
        "tonnes: 0.2008\n",
        "wheat: 0.1691\n",
        "for: 0.1374\n",
        "load: 0.0529\n",
        "soviet: 0.0529\n",
        "barley: 0.0423\n",
        "march: 0.0317\n",
        "sources: 0.0317\n",
        "another: 0.0317\n",
        "algeria: 0.0212\n",
        "colza: 0.0212\n",
        "ship: 0.0212\n",
        "through: 0.0212\n",
        "saudi: 0.0212\n",
        "arabia: 0.0212\n",
        "mauritania: 0.0106\n",
        "morocco: 0.0106\n",
        "greece: 0.0106\n",
        "equivalent: 0.0106\n",
        "cereal: 0.0106\n",
        "\n",
        "-- label 17 : interest\n",
        "pct: 0.0542\n",
        "to: 0.0542\n",
        "bank: 0.0542\n",
        "savings: 0.0361\n",
        "rates: 0.0361\n",
        "from: 0.0316\n",
        "banks: 0.0271\n",
        "rate: 0.0271\n",
        "funds: 0.0271\n",
        "customers: 0.0181\n",
        "f: 0.0181\n",
        "growth: 0.0181\n",
        "ag: 0.0181\n",
        "lt: 0.0181\n",
        "central: 0.0181\n",
        "money: 0.0181\n",
        "lending: 0.0135\n",
        "cutting: 0.0135\n",
        "cd: 0.0135\n",
        "move: 0.0135\n",
        "\n",
        "-- label 18 : crude\n",
        "opec: 0.0476\n",
        "oil: 0.0404\n",
        "bpd: 0.0309\n",
        "output: 0.0285\n",
        "was: 0.0285\n",
        "ecuador: 0.0214\n",
        "pipeline: 0.0190\n",
        "crude: 0.0190\n",
        "lukman: 0.0166\n",
        "february: 0.0166\n",
        "platform: 0.0143\n",
        "country: 0.0119\n",
        "escalation: 0.0095\n",
        "resume: 0.0095\n",
        "war: 0.0095\n",
        "limited: 0.0095\n",
        "his: 0.0095\n",
        "nigeria: 0.0095\n",
        "agrio: 0.0071\n",
        "jungle: 0.0071\n",
        "\n",
        "-- label 19 : wheat\n",
        "wheat: 0.1174\n",
        "department: 0.0839\n",
        "australian: 0.0839\n",
        "export: 0.0671\n",
        "tonnes: 0.0503\n",
        "gmp: 0.0336\n",
        "awb: 0.0336\n",
        "world: 0.0336\n",
        "breakeven: 0.0336\n",
        "government: 0.0336\n",
        "benefit: 0.0168\n",
        "receive: 0.0168\n",
        "subsidies: 0.0168\n",
        "low: 0.0168\n",
        "cover: 0.0168\n",
        "report: 0.0168\n",
        "sharp: 0.0168\n",
        "decline: 0.0168\n",
        "usda: 0.0168\n",
        "drop: 0.0168\n",
        "\n",
        "-- label 20 : carcass\n",
        "pork: 0.1124\n",
        "when: 0.0562\n",
        "mandatory: 0.0562\n",
        "controls: 0.0562\n",
        "debated: 0.0562\n",
        "apc: 0.0562\n",
        "delegates: 0.0562\n",
        "decide: 0.0281\n",
        "wednesday: 0.0281\n",
        "producers: 0.0281\n",
        "council: 0.0281\n",
        "listened: 0.0281\n",
        "sides: 0.0281\n",
        "indianapolis: 0.0281\n",
        "iowa: 0.0281\n",
        "former: 0.0281\n",
        "convention: 0.0281\n",
        "assistant: 0.0281\n",
        "usda: 0.0281\n",
        "lesher: 0.0281\n",
        "\n",
        "-- label 21 : corn\n",
        "output: 0.1852\n",
        "tonnes: 0.1389\n",
        "ivory: 0.0926\n",
        "boosting: 0.0463\n",
        "detail: 0.0463\n",
        "sufficiency: 0.0463\n",
        "annually: 0.0463\n",
        "self: 0.0463\n",
        "steadily: 0.0463\n",
        "fraternite: 0.0463\n",
        "decades: 0.0463\n",
        "rapidly: 0.0463\n",
        "boosts: 0.0463\n",
        "bryson: 0.0000\n",
        "provides: 0.0000\n",
        "believes: 0.0000\n",
        "expand: 0.0000\n",
        "investments: 0.0000\n",
        "exploration: 0.0000\n",
        "paid: 0.0000\n",
        "\n",
        "-- label 22 : gnp\n",
        "tax: 0.0536\n",
        "deficit: 0.0453\n",
        "budget: 0.0453\n",
        "sprinkel: 0.0371\n",
        "spending: 0.0330\n",
        "penner: 0.0330\n",
        "economic: 0.0247\n",
        "cut: 0.0247\n",
        "government: 0.0206\n",
        "gnp: 0.0206\n",
        "fiscal: 0.0206\n",
        "billion: 0.0206\n",
        "federal: 0.0165\n",
        "targets: 0.0124\n",
        "process: 0.0124\n",
        "reform: 0.0124\n",
        "looks: 0.0124\n",
        "growth: 0.0124\n",
        "pretty: 0.0082\n",
        "disagree: 0.0082\n",
        "\n",
        "-- label 23 : oilseed\n",
        "scheme: 0.0368\n",
        "britain: 0.0368\n",
        "germany: 0.0368\n",
        "criticism: 0.0245\n",
        "journalists: 0.0245\n",
        "fats: 0.0245\n",
        "butter: 0.0245\n",
        "executive: 0.0245\n",
        "proposals: 0.0245\n",
        "program: 0.0245\n",
        "hit: 0.0245\n",
        "billion: 0.0245\n",
        "radical: 0.0123\n",
        "proposal: 0.0123\n",
        "letter: 0.0123\n",
        "dismantle: 0.0123\n",
        "border: 0.0123\n",
        "less: 0.0123\n",
        "harsh: 0.0123\n",
        "denmark: 0.0123\n",
        "\n",
        "-- label 24 : rapeseed\n",
        "union: 0.1295\n",
        "exports: 0.0777\n",
        "period: 0.0777\n",
        "italy: 0.0777\n",
        "rouen: 0.0777\n",
        "ships: 0.0777\n",
        "loading: 0.0777\n",
        "french: 0.0518\n",
        "took: 0.0518\n",
        "cereals: 0.0518\n",
        "flour: 0.0259\n",
        "tanzania: 0.0259\n",
        "sudan: 0.0259\n",
        "bangladesh: 0.0259\n",
        "africa: 0.0259\n",
        "cyprus: 0.0259\n",
        "brazil: 0.0259\n",
        "dixons: 0.0000\n",
        "bryson: 0.0000\n",
        "stockholders: 0.0000\n",
        "\n",
        "-- label 25 : cpi\n",
        "in: 0.1797\n",
        "pct: 0.0951\n",
        "february: 0.0634\n",
        "may: 0.0529\n",
        "prices: 0.0423\n",
        "january: 0.0423\n",
        "inflation: 0.0317\n",
        "ec: 0.0317\n",
        "year: 0.0317\n",
        "april: 0.0317\n",
        "annual: 0.0317\n",
        "consumer: 0.0317\n",
        "rise: 0.0212\n",
        "statistics: 0.0212\n",
        "istat: 0.0212\n",
        "compared: 0.0212\n",
        "index: 0.0212\n",
        "italy: 0.0212\n",
        "eurostat: 0.0212\n",
        "workers: 0.0106\n",
        "\n",
        "-- label 26 : earn\n",
        "vs: 0.1068\n",
        "mln: 0.0795\n",
        "cts: 0.0644\n",
        "net: 0.0553\n",
        "dlrs: 0.0386\n",
        "loss: 0.0379\n",
        "shr: 0.0333\n",
        "lt: 0.0288\n",
        "revs: 0.0212\n",
        "qtr: 0.0204\n",
        "inc: 0.0174\n",
        "year: 0.0144\n",
        "profit: 0.0136\n",
        "note: 0.0129\n",
        "of: 0.0098\n",
        "includes: 0.0098\n",
        "for: 0.0091\n",
        "billion: 0.0091\n",
        "mths: 0.0091\n",
        "six: 0.0083\n",
        "\n",
        "-- label 27 : bop\n",
        "deficit: 0.1504\n",
        "qtr: 0.1504\n",
        "billion: 0.1504\n",
        "canada: 0.0752\n",
        "current: 0.0752\n",
        "dlrs: 0.0752\n",
        "vs: 0.0752\n",
        "official: 0.0752\n",
        "account: 0.0752\n",
        "montreal: 0.0000\n",
        "dixons: 0.0000\n",
        "bryson: 0.0000\n",
        "pays: 0.0000\n",
        "cenergy: 0.0000\n",
        "gas: 0.0000\n",
        "investments: 0.0000\n",
        "exploration: 0.0000\n",
        "expand: 0.0000\n",
        "believes: 0.0000\n",
        "opportunity: 0.0000\n",
        "\n",
        "-- label 28 : money-supply\n",
        "bank: 0.0985\n",
        "week: 0.0985\n",
        "dlrs: 0.0985\n",
        "m: 0.0493\n",
        "money: 0.0493\n",
        "borrowings: 0.0493\n",
        "day: 0.0493\n",
        "average: 0.0493\n",
        "falls: 0.0493\n",
        "billion: 0.0493\n",
        "canadian: 0.0493\n",
        "canada: 0.0493\n",
        "supply: 0.0493\n",
        "discount: 0.0493\n",
        "fed: 0.0493\n",
        "says: 0.0493\n",
        "investments: 0.0000\n",
        "expand: 0.0000\n",
        "believes: 0.0000\n",
        "gas: 0.0000\n",
        "\n",
        "-- label 29 : acq\n",
        "of: 0.0343\n",
        "lt: 0.0330\n",
        "shares: 0.0330\n",
        "dlrs: 0.0237\n",
        "offer: 0.0198\n",
        "rights: 0.0198\n",
        "corp: 0.0185\n",
        "said: 0.0171\n",
        "it: 0.0171\n",
        "inc: 0.0145\n",
        "network: 0.0132\n",
        "northeast: 0.0132\n",
        "share: 0.0119\n",
        "company: 0.0119\n",
        "bank: 0.0105\n",
        "tender: 0.0105\n",
        "common: 0.0105\n",
        "federal: 0.0105\n",
        "fair: 0.0092\n",
        "right: 0.0092\n",
        "\n",
        "-- label 30 : lumber\n",
        "cip: 0.0620\n",
        "pulp: 0.0620\n",
        "newsprint: 0.0496\n",
        "fibre: 0.0496\n",
        "mill: 0.0496\n",
        "canadian: 0.0372\n",
        "slenniken: 0.0372\n",
        "plant: 0.0372\n",
        "kenaf: 0.0372\n",
        "pacific: 0.0248\n",
        "wood: 0.0248\n",
        "texas: 0.0248\n",
        "begin: 0.0248\n",
        "project: 0.0248\n",
        "artificially: 0.0124\n",
        "grown: 0.0124\n",
        "reaches: 0.0124\n",
        "need: 0.0124\n",
        "feet: 0.0124\n",
        "quality: 0.0124\n",
        "\n",
        "-- label 31 : grain\n",
        "maize: 0.1235\n",
        "china: 0.1029\n",
        "the: 0.0617\n",
        "coast: 0.0617\n",
        "egypt: 0.0412\n",
        "farmers: 0.0412\n",
        "grain: 0.0412\n",
        "turkey: 0.0412\n",
        "country: 0.0412\n",
        "freight: 0.0412\n",
        "matin: 0.0206\n",
        "insufficient: 0.0206\n",
        "ahead: 0.0206\n",
        "recent: 0.0206\n",
        "including: 0.0206\n",
        "reimburse: 0.0206\n",
        "record: 0.0206\n",
        "leading: 0.0206\n",
        "harvested: 0.0206\n",
        "crop: 0.0206\n",
        "\n",
        "-- label 32 : wpi\n",
        "pct: 0.2217\n",
        "prices: 0.1478\n",
        "march: 0.0985\n",
        "producer: 0.0985\n",
        "february: 0.0739\n",
        "fell: 0.0739\n",
        "statistics: 0.0493\n",
        "heating: 0.0246\n",
        "fallen: 0.0246\n",
        "liquefied: 0.0246\n",
        "coffee: 0.0246\n",
        "federal: 0.0246\n",
        "dropped: 0.0246\n",
        "cheaper: 0.0246\n",
        "unsuccessfully: 0.0000\n",
        "gas: 0.0000\n",
        "exploration: 0.0000\n",
        "cenergy: 0.0000\n",
        "pays: 0.0000\n",
        "investments: 0.0000\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}