{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "study\n",
      "study\n",
      "study\n"
     ]
    }
   ],
   "source": [
    "from nltk import WordNetLemmatizer\n",
    "lemma = WordNetLemmatizer()\n",
    "\n",
    "words = [\"study\", \"studying\", \"studied\"]\n",
    "for word in words:\n",
    "    print(lemma.lemmatize(word, pos='v'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['study', 'study', 'study']\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "words = \"Studies Studying Studied\"\n",
    "lemmas=[token.lemma_ for token in nlp(words)]\n",
    "print(lemmas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('Once', 'ADV'), ('upon', 'SCONJ'), ('a', 'DET'), ('time', 'NOUN'), ('there', 'PRON'), ('was', 'VERB'), ('an', 'DET'), ('old', 'ADJ'), ('mother', 'NOUN'), ('pig', 'NOUN'), ('who', 'PRON'), ('had', 'VERB'), ('three', 'NUM'), ('little', 'ADJ'), ('pigs', 'NOUN'), ('and', 'CCONJ'), ('not', 'PART'), ('enough', 'ADJ'), ('food', 'NOUN'), ('to', 'PART'), ('feed', 'VERB'), ('them', 'PRON'), ('.', 'PUNCT'), ('\\n', 'SPACE'), ('So', 'ADV'), ('when', 'SCONJ'), ('they', 'PRON'), ('were', 'AUX'), ('old', 'ADJ'), ('enough', 'ADV'), (',', 'PUNCT'), ('she', 'PRON'), ('sent', 'VERB'), ('them', 'PRON'), ('out', 'ADP'), ('into', 'ADP'), ('the', 'DET'), ('world', 'NOUN'), ('to', 'PART'), ('seek', 'VERB'), ('their', 'PRON'), ('fortunes', 'NOUN'), ('.', 'PUNCT')]\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    " \n",
    "#load the small English model\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "sentence = \"\"\"Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. \n",
    "So when they were old enough, she sent them out into the world to seek their fortunes. \n",
    "The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. \n",
    "The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. \n",
    "Then, they sang and danced and played together the rest of the day. \n",
    "The third little pig worked hard all day and built his house with bricks. \n",
    "It was a sturdy house complete with a fine fireplace and chimney. \n",
    "It looked like it could withstand the strongest winds.\"\"\"\n",
    "\n",
    "pos = [(token.text, token.pos_) for token in nlp(sentence)]\n",
    "print(pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(S\n",
      "  (NP Machine/NN learning/NN)\n",
      "  is/VBZ\n",
      "  (NP a/DT field/NN)\n",
      "  of/IN\n",
      "  (NP study/NN)\n",
      "  in/IN\n",
      "  (NP artificial/JJ intelligence/NN)\n",
      "  ./.)\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "content = \"Machine learning is a field of study in artificial intelligence.\"\n",
    "tokenized_text = nltk.word_tokenize(content)\n",
    "tagged_token = nltk.pos_tag(tokenized_text)\n",
    "grammer = r\"\"\"\n",
    "            NP: {<DT>?<JJ>+<NN>}\n",
    "            {<DT>+<NN>}\n",
    "            {<NN>+}\n",
    "            \"\"\"\n",
    "phrases = nltk.RegexpParser(grammer)\n",
    "result = phrases.parse(tagged_token)\n",
    "print(result)\n",
    "result.draw()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" xml:lang=\"en\" id=\"4b1cdc0a1aa14c3587b0c4daf50e929d-0\" class=\"displacy\" width=\"575\" height=\"224.5\" direction=\"ltr\" style=\"max-width: none; height: 224.5px; color: #000000; background: #ffffff; font-family: Arial; direction: ltr\">\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"134.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"50\">Bob</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"50\">PROPN</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"134.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"225\">ate</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"225\">VERB</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"134.5\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"400\">apple.</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"400\">NOUN</tspan>\n",
       "</text>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-4b1cdc0a1aa14c3587b0c4daf50e929d-0-0\" stroke-width=\"2px\" d=\"M70,89.5 C70,2.0 225.0,2.0 225.0,89.5\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-4b1cdc0a1aa14c3587b0c4daf50e929d-0-0\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">nsubj</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M70,91.5 L62,79.5 78,79.5\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-4b1cdc0a1aa14c3587b0c4daf50e929d-0-1\" stroke-width=\"2px\" d=\"M245,89.5 C245,2.0 400.0,2.0 400.0,89.5\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-4b1cdc0a1aa14c3587b0c4daf50e929d-0-1\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">dobj</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M400.0,91.5 L408.0,79.5 392.0,79.5\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "</svg></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import spacy\n",
    "from spacy import displacy\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "doc = nlp(\"Bob ate apple.\")\n",
    "# Since this is an interactive Jupyter environment, we can use displacy.render here\n",
    "displacy.render(doc, style='dep')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLTK sentence segmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them.\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import sent_tokenize\n",
    "text = \"\"\"Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. \n",
    "So when they were old enough, she sent them out into the world to seek their fortunes. \n",
    "The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. \n",
    "The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. \n",
    "Then, they sang and danced and played together the rest of the day. \n",
    "The third little pig worked hard all day and built his house with bricks. \n",
    "It was a sturdy house complete with a fine fireplace and chimney. \n",
    "It looked like it could withstand the strongest winds.\"\"\"\n",
    "tokenized_text=sent_tokenize(text)\n",
    "print(tokenized_text[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# spacy sentence segmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them.\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "nlp = spacy.load('en_core_web_sm') # or whatever model you have installed\n",
    "\n",
    "text = \"\"\"Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. \n",
    "So when they were old enough, she sent them out into the world to seek their fortunes. \n",
    "The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. \n",
    "The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. \n",
    "Then, they sang and danced and played together the rest of the day. \n",
    "The third little pig worked hard all day and built his house with bricks. \n",
    "It was a sturdy house complete with a fine fireplace and chimney. \n",
    "It looked like it could withstand the strongest winds.\"\"\"\n",
    "\n",
    "doc = nlp(text)\n",
    "sentences = [sent.text.strip() for sent in doc.sents]\n",
    "print(sentences[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLTK word tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Once', 'upon', 'a', 'time', 'there', 'was', 'an', 'old', 'mother', 'pig', 'who', 'had', 'three', 'little', 'pigs', 'and', 'not', 'enough', 'food', 'to', 'feed', 'them', '.', 'So', 'when', 'they', 'were', 'old', 'enough', ',', 'she', 'sent', 'them', 'out', 'into', 'the', 'world', 'to', 'seek', 'their', 'fortunes', '.', 'The', 'first', 'little', 'pig', 'was', 'very', 'lazy', '.', 'He', 'did', \"n't\", 'want', 'to', 'work', 'at', 'all', 'and', 'he', 'built', 'his', 'house', 'out', 'of', 'straw', '.', 'The', 'second', 'little', 'pig', 'worked', 'a', 'little', 'bit', 'harder', 'but', 'he', 'was', 'somewhat', 'lazy', 'too', 'and', 'he', 'built', 'his', 'house', 'out', 'of', 'sticks', '.', 'Then', ',', 'they', 'sang', 'and', 'danced', 'and', 'played', 'together', 'the', 'rest', 'of', 'the', 'day', '.', 'The', 'third', 'little', 'pig', 'worked', 'hard', 'all', 'day', 'and', 'built', 'his', 'house', 'with', 'bricks', '.', 'It', 'was', 'a', 'sturdy', 'house', 'complete', 'with', 'a', 'fine', 'fireplace', 'and', 'chimney', '.', 'It', 'looked', 'like', 'it', 'could', 'withstand', 'the', 'strongest', 'winds', '.']\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "text = \"\"\"Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. \n",
    "So when they were old enough, she sent them out into the world to seek their fortunes. \n",
    "The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. \n",
    "The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. \n",
    "Then, they sang and danced and played together the rest of the day. \n",
    "The third little pig worked hard all day and built his house with bricks. \n",
    "It was a sturdy house complete with a fine fireplace and chimney. \n",
    "It looked like it could withstand the strongest winds.\"\"\"\n",
    "tokenized_text=word_tokenize(text)\n",
    "print(tokenized_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# spacy word tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Once', 'upon', 'a', 'time', 'there', 'was', 'an', 'old', 'mother', 'pig', 'who', 'had', 'three', 'little', 'pigs', 'and', 'not', 'enough', 'food', 'to', 'feed', 'them', '.', '\\n', 'So', 'when', 'they', 'were', 'old', 'enough', ',', 'she', 'sent', 'them', 'out', 'into', 'the', 'world', 'to', 'seek', 'their', 'fortunes', '.', '\\n', 'The', 'first', 'little', 'pig', 'was', 'very', 'lazy', '.', 'He', 'did', \"n't\", 'want', 'to', 'work', 'at', 'all', 'and', 'he', 'built', 'his', 'house', 'out', 'of', 'straw', '.', '\\n', 'The', 'second', 'little', 'pig', 'worked', 'a', 'little', 'bit', 'harder', 'but', 'he', 'was', 'somewhat', 'lazy', 'too', 'and', 'he', 'built', 'his', 'house', 'out', 'of', 'sticks', '.', '\\n', 'Then', ',', 'they', 'sang', 'and', 'danced', 'and', 'played', 'together', 'the', 'rest', 'of', 'the', 'day', '.', '\\n', 'The', 'third', 'little', 'pig', 'worked', 'hard', 'all', 'day', 'and', 'built', 'his', 'house', 'with', 'bricks', '.', '\\n', 'It', 'was', 'a', 'sturdy', 'house', 'complete', 'with', 'a', 'fine', 'fireplace', 'and', 'chimney', '.', '\\n', 'It', 'looked', 'like', 'it', 'could', 'withstand', 'the', 'strongest', 'winds', '.']\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    " \n",
    "#load the small English model\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "sentence = \"\"\"Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. \n",
    "So when they were old enough, she sent them out into the world to seek their fortunes. \n",
    "The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. \n",
    "The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. \n",
    "Then, they sang and danced and played together the rest of the day. \n",
    "The third little pig worked hard all day and built his house with bricks. \n",
    "It was a sturdy house complete with a fine fireplace and chimney. \n",
    "It looked like it could withstand the strongest winds.\"\"\"\n",
    "doc = nlp(sentence)\n",
    "tokens = [token.text for token in doc]\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLTK stop words removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['upon', 'time', 'old', 'mother', 'pig', 'three', 'little', 'pigs', 'enough', 'food', 'feed', '.', 'old', 'enough', ',', 'sent', 'world', 'seek', 'fortunes', '.', 'first', 'little', 'pig', 'lazy', '.', \"n't\", 'want', 'work', 'built', 'house', 'straw', '.', 'second', 'little', 'pig', 'worked', 'little', 'bit', 'harder', 'somewhat', 'lazy', 'built', 'house', 'sticks', '.', ',', 'sang', 'danced', 'played', 'together', 'rest', 'day', '.', 'third', 'little', 'pig', 'worked', 'hard', 'day', 'built', 'house', 'bricks', '.', 'sturdy', 'house', 'complete', 'fine', 'fireplace', 'chimney', '.', 'looked', 'like', 'could', 'withstand', 'strongest', 'winds', '.']\n"
     ]
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "sentence = \"\"\"Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. \n",
    "So when they were old enough, she sent them out into the world to seek their fortunes. \n",
    "The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. \n",
    "The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. \n",
    "Then, they sang and danced and played together the rest of the day. \n",
    "The third little pig worked hard all day and built his house with bricks. \n",
    "It was a sturdy house complete with a fine fireplace and chimney. \n",
    "It looked like it could withstand the strongest winds.\"\"\"\n",
    " \n",
    "stop_words = set(stopwords.words('english'))\n",
    "# print(stop_words)\n",
    " \n",
    "word_tokens = word_tokenize(sentence)\n",
    "filtered_tokens = [word for word in word_tokens if not word.lower() in stop_words]\n",
    "print(filtered_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# spacy stop words removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['time', 'old', 'mother', 'pig', 'little', 'pigs', 'food', 'feed', '.', '\\n', 'old', ',', 'sent', 'world', 'seek', 'fortunes', '.', '\\n', 'little', 'pig', 'lazy', '.', 'want', 'work', 'built', 'house', 'straw', '.', '\\n', 'second', 'little', 'pig', 'worked', 'little', 'bit', 'harder', 'somewhat', 'lazy', 'built', 'house', 'sticks', '.', '\\n', ',', 'sang', 'danced', 'played', 'rest', 'day', '.', '\\n', 'little', 'pig', 'worked', 'hard', 'day', 'built', 'house', 'bricks', '.', '\\n', 'sturdy', 'house', 'complete', 'fine', 'fireplace', 'chimney', '.', '\\n', 'looked', 'like', 'withstand', 'strongest', 'winds', '.']\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    " \n",
    "#load the small English model\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "sentence = \"\"\"Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. \n",
    "So when they were old enough, she sent them out into the world to seek their fortunes. \n",
    "The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. \n",
    "The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. \n",
    "Then, they sang and danced and played together the rest of the day. \n",
    "The third little pig worked hard all day and built his house with bricks. \n",
    "It was a sturdy house complete with a fine fireplace and chimney. \n",
    "It looked like it could withstand the strongest winds.\"\"\"\n",
    "\n",
    "doc = nlp(sentence)\n",
    "filtered_tokens = [token.text for token in doc if not token.is_stop]\n",
    "print(filtered_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLTK Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['onc', 'upon', 'a', 'time', 'there', 'wa', 'an', 'old', 'mother', 'pig', 'who', 'had', 'three', 'littl', 'pig', 'and', 'not', 'enough', 'food', 'to', 'feed', 'them', '.', 'So', 'when', 'they', 'were', 'old', 'enough', ',', 'she', 'sent', 'them', 'out', 'into', 'the', 'world', 'to', 'seek', 'their', 'fortun', '.', 'the', 'first', 'littl', 'pig', 'wa', 'veri', 'lazi', '.', 'He', 'did', \"n't\", 'want', 'to', 'work', 'at', 'all', 'and', 'he', 'built', 'hi', 'hous', 'out', 'of', 'straw', '.', 'the', 'second', 'littl', 'pig', 'work', 'a', 'littl', 'bit', 'harder', 'but', 'he', 'wa', 'somewhat', 'lazi', 'too', 'and', 'he', 'built', 'hi', 'hous', 'out', 'of', 'stick', '.', 'then', ',', 'they', 'sang', 'and', 'danc', 'and', 'play', 'togeth', 'the', 'rest', 'of', 'the', 'day', '.', 'the', 'third', 'littl', 'pig', 'work', 'hard', 'all', 'day', 'and', 'built', 'hi', 'hous', 'with', 'brick', '.', 'It', 'wa', 'a', 'sturdi', 'hous', 'complet', 'with', 'a', 'fine', 'fireplac', 'and', 'chimney', '.', 'It', 'look', 'like', 'it', 'could', 'withstand', 'the', 'strongest', 'wind', '.']\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import PorterStemmer\n",
    "sentence = \"\"\"Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. \n",
    "So when they were old enough, she sent them out into the world to seek their fortunes. \n",
    "The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. \n",
    "The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. \n",
    "Then, they sang and danced and played together the rest of the day. \n",
    "The third little pig worked hard all day and built his house with bricks. \n",
    "It was a sturdy house complete with a fine fireplace and chimney. \n",
    "It looked like it could withstand the strongest winds.\"\"\"\n",
    "ps = PorterStemmer()\n",
    "word_tokens = word_tokenize(sentence)\n",
    "stem_tokens = [ps.stem(word) for word in word_tokens]\n",
    "print(stem_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# spacy Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# no stemming in spacy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLTK POS tagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('Once', 'RB'), ('upon', 'IN'), ('a', 'DT'), ('time', 'NN'), ('there', 'EX'), ('was', 'VBD'), ('an', 'DT'), ('old', 'JJ'), ('mother', 'NN'), ('pig', 'NN'), ('who', 'WP'), ('had', 'VBD'), ('three', 'CD'), ('little', 'JJ'), ('pigs', 'NNS'), ('and', 'CC'), ('not', 'RB'), ('enough', 'RB'), ('food', 'NN'), ('to', 'TO'), ('feed', 'VB'), ('them', 'PRP'), ('.', '.'), ('So', 'RB'), ('when', 'WRB'), ('they', 'PRP'), ('were', 'VBD'), ('old', 'JJ'), ('enough', 'RB'), (',', ','), ('she', 'PRP'), ('sent', 'VBD'), ('them', 'PRP'), ('out', 'RP'), ('into', 'IN'), ('the', 'DT'), ('world', 'NN'), ('to', 'TO'), ('seek', 'VB'), ('their', 'PRP$'), ('fortunes', 'NNS'), ('.', '.'), ('The', 'DT'), ('first', 'JJ'), ('little', 'JJ'), ('pig', 'NN'), ('was', 'VBD'), ('very', 'RB'), ('lazy', 'JJ'), ('.', '.'), ('He', 'PRP'), ('did', 'VBD'), (\"n't\", 'RB'), ('want', 'VB'), ('to', 'TO'), ('work', 'VB'), ('at', 'IN'), ('all', 'DT'), ('and', 'CC'), ('he', 'PRP'), ('built', 'VBD'), ('his', 'PRP$'), ('house', 'NN'), ('out', 'IN'), ('of', 'IN'), ('straw', 'NN'), ('.', '.'), ('The', 'DT'), ('second', 'JJ'), ('little', 'JJ'), ('pig', 'NN'), ('worked', 'VBD'), ('a', 'DT'), ('little', 'JJ'), ('bit', 'NN'), ('harder', 'RBR'), ('but', 'CC'), ('he', 'PRP'), ('was', 'VBD'), ('somewhat', 'RB'), ('lazy', 'JJ'), ('too', 'RB'), ('and', 'CC'), ('he', 'PRP'), ('built', 'VBD'), ('his', 'PRP$'), ('house', 'NN'), ('out', 'IN'), ('of', 'IN'), ('sticks', 'NNS'), ('.', '.'), ('Then', 'RB'), (',', ','), ('they', 'PRP'), ('sang', 'VBD'), ('and', 'CC'), ('danced', 'VBD'), ('and', 'CC'), ('played', 'VBD'), ('together', 'RB'), ('the', 'DT'), ('rest', 'NN'), ('of', 'IN'), ('the', 'DT'), ('day', 'NN'), ('.', '.'), ('The', 'DT'), ('third', 'JJ'), ('little', 'JJ'), ('pig', 'NN'), ('worked', 'VBD'), ('hard', 'JJ'), ('all', 'DT'), ('day', 'NN'), ('and', 'CC'), ('built', 'VBD'), ('his', 'PRP$'), ('house', 'NN'), ('with', 'IN'), ('bricks', 'NNS'), ('.', '.'), ('It', 'PRP'), ('was', 'VBD'), ('a', 'DT'), ('sturdy', 'JJ'), ('house', 'NN'), ('complete', 'JJ'), ('with', 'IN'), ('a', 'DT'), ('fine', 'JJ'), ('fireplace', 'NN'), ('and', 'CC'), ('chimney', 'NN'), ('.', '.'), ('It', 'PRP'), ('looked', 'VBD'), ('like', 'IN'), ('it', 'PRP'), ('could', 'MD'), ('withstand', 'VB'), ('the', 'DT'), ('strongest', 'JJS'), ('winds', 'NNS'), ('.', '.')]\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "from nltk import pos_tag\n",
    "sentence = \"\"\"Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. \n",
    "So when they were old enough, she sent them out into the world to seek their fortunes. \n",
    "The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. \n",
    "The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. \n",
    "Then, they sang and danced and played together the rest of the day. \n",
    "The third little pig worked hard all day and built his house with bricks. \n",
    "It was a sturdy house complete with a fine fireplace and chimney. \n",
    "It looked like it could withstand the strongest winds.\"\"\"\n",
    "word_tokens = word_tokenize(sentence)\n",
    "pos_tagging = pos_tag(word_tokens)\n",
    "print(pos_tagging)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# spacy POS tagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('Once', 'ADV'), ('upon', 'SCONJ'), ('a', 'DET'), ('time', 'NOUN'), ('there', 'PRON'), ('was', 'VERB'), ('an', 'DET'), ('old', 'ADJ'), ('mother', 'NOUN'), ('pig', 'NOUN'), ('who', 'PRON'), ('had', 'VERB'), ('three', 'NUM'), ('little', 'ADJ'), ('pigs', 'NOUN'), ('and', 'CCONJ'), ('not', 'PART'), ('enough', 'ADJ'), ('food', 'NOUN'), ('to', 'PART'), ('feed', 'VERB'), ('them', 'PRON'), ('.', 'PUNCT'), ('\\n', 'SPACE'), ('So', 'ADV'), ('when', 'SCONJ'), ('they', 'PRON'), ('were', 'AUX'), ('old', 'ADJ'), ('enough', 'ADV'), (',', 'PUNCT'), ('she', 'PRON'), ('sent', 'VERB'), ('them', 'PRON'), ('out', 'ADP'), ('into', 'ADP'), ('the', 'DET'), ('world', 'NOUN'), ('to', 'PART'), ('seek', 'VERB'), ('their', 'PRON'), ('fortunes', 'NOUN'), ('.', 'PUNCT'), ('\\n', 'SPACE'), ('The', 'DET'), ('first', 'ADJ'), ('little', 'ADJ'), ('pig', 'NOUN'), ('was', 'AUX'), ('very', 'ADV'), ('lazy', 'ADJ'), ('.', 'PUNCT'), ('He', 'PRON'), ('did', 'AUX'), (\"n't\", 'PART'), ('want', 'VERB'), ('to', 'PART'), ('work', 'VERB'), ('at', 'ADV'), ('all', 'ADV'), ('and', 'CCONJ'), ('he', 'PRON'), ('built', 'VERB'), ('his', 'PRON'), ('house', 'NOUN'), ('out', 'ADP'), ('of', 'ADP'), ('straw', 'NOUN'), ('.', 'PUNCT'), ('\\n', 'SPACE'), ('The', 'DET'), ('second', 'ADJ'), ('little', 'ADJ'), ('pig', 'NOUN'), ('worked', 'VERB'), ('a', 'DET'), ('little', 'ADJ'), ('bit', 'NOUN'), ('harder', 'ADV'), ('but', 'CCONJ'), ('he', 'PRON'), ('was', 'AUX'), ('somewhat', 'ADV'), ('lazy', 'ADJ'), ('too', 'ADV'), ('and', 'CCONJ'), ('he', 'PRON'), ('built', 'VERB'), ('his', 'PRON'), ('house', 'NOUN'), ('out', 'ADP'), ('of', 'ADP'), ('sticks', 'NOUN'), ('.', 'PUNCT'), ('\\n', 'SPACE'), ('Then', 'ADV'), (',', 'PUNCT'), ('they', 'PRON'), ('sang', 'VERB'), ('and', 'CCONJ'), ('danced', 'VERB'), ('and', 'CCONJ'), ('played', 'VERB'), ('together', 'ADV'), ('the', 'DET'), ('rest', 'NOUN'), ('of', 'ADP'), ('the', 'DET'), ('day', 'NOUN'), ('.', 'PUNCT'), ('\\n', 'SPACE'), ('The', 'DET'), ('third', 'ADJ'), ('little', 'ADJ'), ('pig', 'NOUN'), ('worked', 'VERB'), ('hard', 'ADV'), ('all', 'DET'), ('day', 'NOUN'), ('and', 'CCONJ'), ('built', 'VERB'), ('his', 'PRON'), ('house', 'NOUN'), ('with', 'ADP'), ('bricks', 'NOUN'), ('.', 'PUNCT'), ('\\n', 'SPACE'), ('It', 'PRON'), ('was', 'AUX'), ('a', 'DET'), ('sturdy', 'ADJ'), ('house', 'NOUN'), ('complete', 'ADJ'), ('with', 'ADP'), ('a', 'DET'), ('fine', 'ADJ'), ('fireplace', 'NOUN'), ('and', 'CCONJ'), ('chimney', 'NOUN'), ('.', 'PUNCT'), ('\\n', 'SPACE'), ('It', 'PRON'), ('looked', 'VERB'), ('like', 'SCONJ'), ('it', 'PRON'), ('could', 'AUX'), ('withstand', 'VERB'), ('the', 'DET'), ('strongest', 'ADJ'), ('winds', 'NOUN'), ('.', 'PUNCT')]\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "sentence = \"\"\"Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. \n",
    "So when they were old enough, she sent them out into the world to seek their fortunes. \n",
    "The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. \n",
    "The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. \n",
    "Then, they sang and danced and played together the rest of the day. \n",
    "The third little pig worked hard all day and built his house with bricks. \n",
    "It was a sturdy house complete with a fine fireplace and chimney. \n",
    "It looked like it could withstand the strongest winds.\"\"\"\n",
    "doc = nlp(sentence)\n",
    "pos_tagging = [(token.text, token.pos_) for token in doc]\n",
    "print(pos_tagging)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLTK Lemmatization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Studying', 'is', 'important', 'to', 'pas', 'your', 'exam', '.']\n",
      "['Studying', 'be', 'important', 'to', 'pass', 'your', 'exam', '.']\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import wordnet\n",
    "def get_wordnet_pos(treebank_tag):\n",
    "        \"\"\"\n",
    "        return WORDNET POS compliance to WORDENT lemmatization (a,n,r,v) \n",
    "        \"\"\"\n",
    "        if treebank_tag.startswith('J'):\n",
    "            return wordnet.ADJ\n",
    "        elif treebank_tag.startswith('V'):\n",
    "            return wordnet.VERB\n",
    "        elif treebank_tag.startswith('N'):\n",
    "            return wordnet.NOUN\n",
    "        elif treebank_tag.startswith('R'):\n",
    "            return wordnet.ADV\n",
    "        else:\n",
    "            # As default pos in lemmatization is Noun\n",
    "            return wordnet.NOUN\n",
    "\n",
    "from nltk import pos_tag\n",
    "sentence = \"\"\"Studying is important to pass your exam.\"\"\"\n",
    "# sentence = \"\"\"I saw Alice, she was studying.\"\"\"\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "word_tokens = word_tokenize(sentence)\n",
    "pos_tags = pos_tag(word_tokens)\n",
    "lemma_tokens = [lemmatizer.lemmatize(word) for word, tag in pos_tags]\n",
    "print(lemma_tokens)\n",
    "lemma_tokens = [lemmatizer.lemmatize(word, pos=get_wordnet_pos(tag)) for word, tag in pos_tags]\n",
    "print(lemma_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLTK Lemmatization POS tagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import wordnet\n",
    "def get_wordnet_pos(treebank_tag):\n",
    "        \"\"\"\n",
    "        return WORDNET POS compliance to WORDENT lemmatization (a,n,r,v) \n",
    "        \"\"\"\n",
    "        if treebank_tag.startswith('J'):\n",
    "            return wordnet.ADJ\n",
    "        elif treebank_tag.startswith('V'):\n",
    "            return wordnet.VERB\n",
    "        elif treebank_tag.startswith('N'):\n",
    "            return wordnet.NOUN\n",
    "        elif treebank_tag.startswith('R'):\n",
    "            return wordnet.ADV\n",
    "        else:\n",
    "            # As default pos in lemmatization is Noun\n",
    "            return wordnet.NOUN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Once', 'upon', 'a', 'time', 'there', 'be', 'an', 'old', 'mother', 'pig', 'who', 'have', 'three', 'little', 'pig', 'and', 'not', 'enough', 'food', 'to', 'fee', 'them', '.', 'So', 'when', 'they', 'be', 'old', 'enough', ',', 'she', 'send', 'them', 'out', 'into', 'the', 'world', 'to', 'seek', 'their', 'fortune', '.', 'The', 'first', 'little', 'pig', 'be', 'very', 'lazy', '.', 'He', 'do', \"n't\", 'want', 'to', 'work', 'at', 'all', 'and', 'he', 'build', 'his', 'house', 'out', 'of', 'straw', '.', 'The', 'second', 'little', 'pig', 'work', 'a', 'little', 'bit', 'hard', 'but', 'he', 'be', 'somewhat', 'lazy', 'too', 'and', 'he', 'build', 'his', 'house', 'out', 'of', 'stick', '.', 'Then', ',', 'they', 'sing', 'and', 'dance', 'and', 'play', 'together', 'the', 'rest', 'of', 'the', 'day', '.', 'The', 'third', 'little', 'pig', 'work', 'hard', 'all', 'day', 'and', 'build', 'his', 'house', 'with', 'brick', '.', 'It', 'be', 'a', 'sturdy', 'house', 'complete', 'with', 'a', 'fine', 'fireplace', 'and', 'chimney', '.', 'It', 'look', 'like', 'it', 'could', 'withstand', 'the', 'strong', 'wind', '.']\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk import pos_tag\n",
    "sentence = \"\"\"Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. \n",
    "So when they were old enough, she sent them out into the world to seek their fortunes. \n",
    "The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. \n",
    "The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. \n",
    "Then, they sang and danced and played together the rest of the day. \n",
    "The third little pig worked hard all day and built his house with bricks. \n",
    "It was a sturdy house complete with a fine fireplace and chimney. \n",
    "It looked like it could withstand the strongest winds.\"\"\"\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "word_tokens = word_tokenize(sentence)\n",
    "pos_tagging = pos_tag(word_tokens)\n",
    "\n",
    "lemma_tokens = [lemmatizer.lemmatize(word, pos=get_wordnet_pos(pos_t)) for word, pos_t in pos_tagging]\n",
    "print(lemma_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# spacy lemmatization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['study', 'be', 'important', 'to', 'pass', 'your', 'exam', '.']\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "sentence = \"\"\"Studying is important to pass your exam.\"\"\"\n",
    "# sentence = \"\"\"I saw Alice, she was studying.\"\"\"\n",
    "doc = nlp(sentence)\n",
    "lemma_tokens = [token.lemma_ for token in doc]\n",
    "print(lemma_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLTK Noun phrase chunking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Machine learning', 'a field', 'study', 'artificial intelligence']\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "from nltk import Tree\n",
    "from nltk.tokenize import word_tokenize\n",
    "sentence = \"\"\"Machine learning is a field of study in artificial intelligence\"\"\"\n",
    "tokenized_text = nltk.word_tokenize(sentence)\n",
    "tagged_token = nltk.pos_tag(tokenized_text)\n",
    "grammer = r\"\"\"\n",
    "            NP: {<DT>?<JJ>+<NN>}\n",
    "            {<DT>+<NN>}\n",
    "            {<NN>+}\n",
    "            \"\"\"\n",
    "phrases = nltk.RegexpParser(grammer)\n",
    "result = phrases.parse(tagged_token)\n",
    "noun_phrases = []\n",
    "for child in result:\n",
    "    if isinstance(child, Tree):               \n",
    "        if child.label() == 'NP':\n",
    "            np = \" \".join([lf[0] for lf in child.leaves()])\n",
    "            noun_phrases.append(np)\n",
    "print(noun_phrases)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# spacy Noun phrase chunking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Machine learning, a field, study, artificial intelligence]\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "sentence =  \"\"\"Machine learning is a field of study in artificial intelligence\"\"\"\n",
    "doc = nlp(sentence)\n",
    "noun_phrases = [np for np in doc.noun_chunks]\n",
    "print(noun_phrases)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLTK Dependency Parsing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.parse.stanford import StanfordDependencyParser\n",
    "# Path to CoreNLP jar unzipped\n",
    "\n",
    "jar_path = '/content/stanford-corenlp-4.2.2/stanford-corenlp-4.2.2.jar'\n",
    "\n",
    "# Path to CoreNLP model jar\n",
    "models_jar_path = '/content/stanford-corenlp-4.2.2-models-english.jar'\n",
    "\n",
    "sentence = 'Deemed universities charge huge fees'\n",
    "\n",
    "# Initialize StanfordDependency Parser from the path\n",
    "parser = StanfordDependencyParser(path_to_jar = jar_path, path_to_models_jar = models_jar_path)\n",
    "\n",
    "# Parse the sentence\n",
    "result = parser.raw_parse(sentence)\n",
    "dependency = result.__next__()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# spacy Dependency Parsing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bob, nsubj, ate\n",
      "ate, ROOT, ate\n",
      "apple, dobj, ate\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "sentence = \"\"\"Bob ate apple\"\"\"\n",
    "doc = nlp(sentence)\n",
    "for token in doc:\n",
    "    print(token.text + \", \" + token.dep_ + \", \" + token.head.text)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLTK Text Preprocessing Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('upon', 'IN'), ('time', 'NN'), ('old', 'JJ'), ('mother', 'NN'), ('pig', 'NN'), ('three', 'CD'), ('little', 'JJ'), ('pig', 'NNS'), ('enough', 'RB'), ('food', 'NN'), ('fee', 'VB'), ('old', 'JJ'), ('enough', 'RB'), ('send', 'VBD'), ('world', 'NN'), ('seek', 'VB'), ('fortune', 'NNS'), ('first', 'JJ'), ('little', 'JJ'), ('pig', 'NN'), ('lazy', 'JJ'), (\"n't\", 'RB'), ('want', 'VB'), ('work', 'VB'), ('build', 'VBD'), ('house', 'NN'), ('straw', 'NN'), ('second', 'JJ'), ('little', 'JJ'), ('pig', 'NN'), ('work', 'VBD'), ('little', 'JJ'), ('bit', 'NN'), ('hard', 'RBR'), ('somewhat', 'RB'), ('lazy', 'JJ'), ('build', 'VBD'), ('house', 'NN'), ('stick', 'NNS'), ('sing', 'VBD'), ('dance', 'VBD'), ('play', 'VBD'), ('together', 'RB'), ('rest', 'NN'), ('day', 'NN'), ('third', 'JJ'), ('little', 'JJ'), ('pig', 'NN'), ('work', 'VBD'), ('hard', 'JJ'), ('day', 'NN'), ('build', 'VBD'), ('house', 'NN'), ('brick', 'NNS'), ('sturdy', 'JJ'), ('house', 'NN'), ('complete', 'JJ'), ('fine', 'JJ'), ('fireplace', 'NN'), ('chimney', 'NN'), ('look', 'VBD'), ('like', 'IN'), ('could', 'MD'), ('withstand', 'VB'), ('strong', 'JJS'), ('wind', 'NNS')]\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import sent_tokenize\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk import pos_tag\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "text = \"\"\"Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. \n",
    "So when they were old enough, she sent them out into the world to seek their fortunes. \n",
    "The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. \n",
    "The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. \n",
    "Then, they sang and danced and played together the rest of the day. \n",
    "The third little pig worked hard all day and built his house with bricks. \n",
    "It was a sturdy house complete with a fine fireplace and chimney. \n",
    "It looked like it could withstand the strongest winds.\"\"\"\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "stop_words = set(stopwords.words('english'))\n",
    "processed_text = []\n",
    "sentences = sent_tokenize(text)\n",
    "for sentence in sentences:\n",
    "    tokens = word_tokenize(sentence)\n",
    "    pos_tagged_tokens = pos_tag(tokens)\n",
    "    for token, tag in pos_tagged_tokens:\n",
    "        if token.lower() in stop_words or token in string.punctuation or token.isdigit():\n",
    "            continue\n",
    "        lemma = lemmatizer.lemmatize(token, pos=get_wordnet_pos(tag))\n",
    "        processed_text.append((lemma, tag))\n",
    "print(processed_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# spaCy Text Preprocessing Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('time', 'NOUN'), ('old', 'ADJ'), ('mother', 'NOUN'), ('pig', 'NOUN'), ('little', 'ADJ'), ('pig', 'NOUN'), ('food', 'NOUN'), ('feed', 'VERB'), ('old', 'ADJ'), ('send', 'VERB'), ('world', 'NOUN'), ('seek', 'VERB'), ('fortune', 'NOUN'), ('little', 'ADJ'), ('pig', 'NOUN'), ('lazy', 'ADJ'), ('want', 'VERB'), ('work', 'VERB'), ('build', 'VERB'), ('house', 'NOUN'), ('straw', 'NOUN'), ('second', 'ADJ'), ('little', 'ADJ'), ('pig', 'NOUN'), ('work', 'VERB'), ('little', 'ADJ'), ('bit', 'NOUN'), ('hard', 'ADV'), ('somewhat', 'ADV'), ('lazy', 'ADJ'), ('build', 'VERB'), ('house', 'NOUN'), ('stick', 'NOUN'), ('sing', 'VERB'), ('dance', 'VERB'), ('play', 'VERB'), ('rest', 'NOUN'), ('day', 'NOUN'), ('little', 'ADJ'), ('pig', 'NOUN'), ('work', 'VERB'), ('hard', 'ADV'), ('day', 'NOUN'), ('build', 'VERB'), ('house', 'NOUN'), ('brick', 'NOUN'), ('sturdy', 'ADJ'), ('house', 'NOUN'), ('complete', 'ADJ'), ('fine', 'ADJ'), ('fireplace', 'NOUN'), ('chimney', 'NOUN'), ('look', 'VERB'), ('like', 'SCONJ'), ('withstand', 'VERB'), ('strong', 'ADJ'), ('wind', 'NOUN')]\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "sentence = \"\"\"Once upon a time there was an old mother pig who had three little pigs and not enough food to feed them. \n",
    "So when they were old enough, she sent them out into the world to seek their fortunes. \n",
    "The first little pig was very lazy. He didn't want to work at all and he built his house out of straw. \n",
    "The second little pig worked a little bit harder but he was somewhat lazy too and he built his house out of sticks. \n",
    "Then, they sang and danced and played together the rest of the day. \n",
    "The third little pig worked hard all day and built his house with bricks. \n",
    "It was a sturdy house complete with a fine fireplace and chimney. \n",
    "It looked like it could withstand the strongest winds.\"\"\"\n",
    "doc = nlp(sentence)\n",
    "\n",
    "processed_text = []\n",
    "for token in doc:\n",
    "    if token.is_stop or token.is_punct or token.is_digit or token.is_space:\n",
    "        continue\n",
    "    processed_text.append((token.lemma_, token.pos_))\n",
    "\n",
    "print(processed_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'chasing', 'leash', 'barked', 'tail', 'barking', 'park', 'owner', '\\n', 'dog'}\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    " \n",
    "#load the small English model\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "sentence = \"\"\"The dog barked in the park on another dog.\n",
    "The owner of the dog put him on the leash since he barked.\n",
    "My dog is barking and chasing its tail.\n",
    "\"\"\"\n",
    "\n",
    "doc = nlp(sentence)\n",
    "filtered_tokens = [token.text for token in doc if not token.is_stop and not token.is_punct]\n",
    "print(set(filtered_tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import string\n",
    "string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
