N-gram analysis and the importance of lemmatization for Turkish.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('1920’de', 'açılmış'),\n",
       " ('1921', 'tarihli'),\n",
       " ('20', 'ocak'),\n",
       " ('23', 'nisan'),\n",
       " ('29', 'ekim'),\n",
       " ('alarak', 'cumhurbaşkanı'),\n",
       " ('altında', 'yaşayan'),\n",
       " ('alçakça', 'önlemler'),\n",
       " ('alınan', 'canın'),\n",
       " ('alınmakta,', 'yurdun'),\n",
       " ('an', 'önce'),\n",
       " ('anlatmaya', 'çalışmış'),\n",
       " ('ateşkes', 'imzalanmış,'),\n",
       " ('ayrıntılı', 'sözlerim'),\n",
       " ('açıklık', 'getirmiştir.')]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "from nltk.collocations import *\n",
    "open_file=open('Nutuk.txt',encoding=\"utf-8\")\n",
    "line=\"\"\n",
    "\n",
    "for val in open_file:\n",
    "    line +=val\n",
    "tokens=line.split()\n",
    "tokens=[token.lower() for token in tokens]\n",
    "bigram_measures = nltk.collocations.BigramAssocMeasures()\n",
    "finder = BigramCollocationFinder.from_words(tokens)\n",
    "finder.nbest(bigram_measures.pmi, 15) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('istemiş,', 'halk'),\n",
       " ('millet', 'meclisi'),\n",
       " ('halk', 'seve'),\n",
       " ('kemal', 'atatürk’ün'),\n",
       " ('büyük', 'millet'),\n",
       " ('mustafa', 'kemal'),\n",
       " ('türkiye', 'büyük'),\n",
       " ('seve', 'seve'),\n",
       " ('kemal', 'atatürk'),\n",
       " ('yeni', 'türkiye'),\n",
       " ('türkiye', 'devletinin'),\n",
       " ('bir', 'devleti'),\n",
       " ('çağdaş', 'bir'),\n",
       " ('bu', 'da'),\n",
       " ('bir', 'ulus')]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finder.apply_freq_filter (2) \n",
    "finder.nbest (bigram_measures.pmi, 15) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('millet', 'meclisi'),\n",
       " ('büyük', 'millet'),\n",
       " ('mustafa', 'kemal'),\n",
       " ('türkiye', 'büyük'),\n",
       " ('bir', 'ulus')]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finder.apply_freq_filter (3) \n",
    "finder.nbest (bigram_measures.pmi, 15) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('mustafa', 'kemal')]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finder.apply_freq_filter (4) \n",
    "finder.nbest (bigram_measures.pmi, 15) "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}