Natural Language Toolkit 简明教程

pip3 install nltk

Import nltk

sudo pip install -U nltk

sudo apt update

sudo apt install python3-pip

conda install -c anaconda nltk

'write'

word_stemmer.stem('eating')

'eat'

import nltk
from nltk.tokenize import word_tokenize
word_tokenize('Tutorialspoint.com provides high quality technical tutorials for free.')

['Tutorialspoint.com', 'provides', 'high', 'quality', 'technical', 'tutorials', 'for', 'free', '.']

import nltk

from nltk.tokenize import TreebankWordTokenizer

Tokenizer_wrd = TreebankWordTokenizer()

Tokenizer_wrd.tokenize(
   'Tutorialspoint.com provides high quality technical tutorials for free.'
)

[
   'Tutorialspoint.com', 'provides', 'high', 'quality',
   'technical', 'tutorials', 'for', 'free', '.'
]

import nltk
from nltk.tokenize import TreebankWordTokenizer
tokenizer_wrd = TreebankWordTokenizer()
tokenizer_wrd.tokenize('Tutorialspoint.com provides high quality technical
tutorials for free.')

[
   'Tutorialspoint.com', 'provides', 'high', 'quality',
   'technical', 'tutorials','for', 'free', '.'
]

import nltk
from nltk.tokenize import word_tokenize
word_tokenize('won’t')

['wo', "n't"]]

from nltk.tokenize import WordPunctTokenizer
tokenizer = WordPunctTokenizer()
tokenizer.tokenize(" I can't allow you to go home early")

['I', 'can', "'", 't', 'allow', 'you', 'to', 'go', 'home', 'early']

import nltk
from nltk.tokenize import sent_tokenize
text = "Let us understand the difference between sentence & word tokenizer.
It is going to be a simple example."
sent_tokenize(text)

[
   "Let us understand the difference between sentence & word tokenizer.",
   'It is going to be a simple example.'
]

import nltk
from nltk.tokenize import RegexpTokenizer
tokenizer = RegexpTokenizer("[\w']+")
tokenizer.tokenize("won't is a contraction.")
tokenizer.tokenize("can't is a contraction.")

["won't", 'is', 'a', 'contraction']
["can't", 'is', 'a', 'contraction']

import nltk
from nltk.tokenize import RegexpTokenizer
tokenizer = RegexpTokenizer('/s+' , gaps = True)
tokenizer.tokenize("won't is a contraction.")

["won't", 'is', 'a', 'contraction']

import nltk
from nltk.tokenize import RegexpTokenizer
tokenizer = RegexpTokenizer('/s+' , gaps = False)
tokenizer.tokenize("won't is a contraction.")

[ ]

Guy: How old are you?
Hipster girl: You know, I never answer that question. Because to me, it's about
how mature you are, you know? I mean, a fourteen year old could be more mature
than a twenty-five year old, right? I'm sorry, I just never answer that question.
Guy: But, uh, you're older than eighteen, right?
Hipster girl: Oh, yeah.

from nltk.tokenize import PunktSentenceTokenizer

from nltk.corpus import webtext

text = webtext.raw('C://Users/Leekha/training_tokenizer.txt')

sent_tokenizer = PunktSentenceTokenizer(text)
sents_1 = sent_tokenizer.tokenize(text)
print(sents_1[0])

White guy: So, do you have any plans for this evening?
print(sents_1[1])
Output:
Asian girl: Yeah, being angry!
print(sents_1[670])
Output:
Guy: A hundred bucks?
print(sents_1[675])
Output:
Girl: But you already have a Big Mac...

from nltk.tokenize import PunktSentenceTokenizer
from nltk.corpus import webtext
text = webtext.raw('C://Users/Leekha/training_tokenizer.txt')
sent_tokenizer = PunktSentenceTokenizer(text)
sents_1 = sent_tokenizer.tokenize(text)
print(sents_1[0])

White guy: So, do you have any plans for this evening?

from nltk.tokenize import sent_tokenize
   from nltk.corpus import webtext
   text = webtext.raw('C://Users/Leekha/training_tokenizer.txt')
sents_2 = sent_tokenize(text)

print(sents_2[0])
Output:

White guy: So, do you have any plans for this evening?
print(sents_2[675])
Output:
Hobo: Y'know what I'd do if I was rich?

from nltk.corpus import stopwords

english_stops = set(stopwords.words('english'))
words = ['I', 'am', 'a', 'writer']
[word for word in words if word not in english_stops]

['I', 'writer']

from nltk.corpus import stopwords
english_stops = set(stopwords.words('english'))
words = ['I', 'am', 'a', 'writer']
[word for word in words if word not in english_stops]

['I', 'writer']

from nltk.corpus import stopwords
stopwords.fileids()

[
   'arabic', 'azerbaijani', 'danish', 'dutch', 'english', 'finnish', 'french',
   'german', 'greek', 'hungarian', 'indonesian', 'italian', 'kazakh', 'nepali',
   'norwegian', 'portuguese', 'romanian', 'russian', 'slovene', 'spanish',
   'swedish', 'tajik', 'turkish'
]

from nltk.corpus import wordnet as wn

syn = wn.synsets('dog')[0]

syn.name()
Output:
'dog.n.01'

syn.definition()
Output:
'a member of the genus Canis (probably descended from the common wolf) that has
been domesticated by man since prehistoric times; occurs in many breeds'

syn.examples()
Output:
['the dog barked all night']

from nltk.corpus import wordnet as wn
syn = wn.synsets('dog')[0]
syn.name()
syn.definition()
syn.examples()

[Synset('canine.n.02'), Synset('domestic_animal.n.01')]

syn.hypernyms()[0].hyponyms()

[
   Synset('bitch.n.04'),
   Synset('dog.n.01'),
   Synset('fox.n.01'),
   Synset('hyena.n.01'),
   Synset('jackal.n.01'),
   Synset('wild_dog.n.01'),
   Synset('wolf.n.01')
]

syn.root_hypernyms()

[Synset('entity.n.01')]

from nltk.corpus import wordnet as wn
syn = wn.synsets('dog')[0]
syn.hypernyms()
syn.hypernyms()[0].hyponyms()
syn.root_hypernyms()

[Synset('entity.n.01')]

from nltk.corpus import wordnet as wn
syn = wn.synsets('dog')[0]
lemmas = syn.lemmas()
len(lemmas)

3

lemmas[0].name()
Output:
'dog'

lemmas[1].name()
Output:
'domestic_dog'

lemmas[2].name()
Output:
'Canis_familiaris'

from nltk.corpus import wordnet as wn
   syn1 = wn.synset('good.n.02')
   antonym1 = syn1.lemmas()[0].antonyms()[0]
antonym1.name()

'evil'

antonym1.synset().definition()

'the quality of being morally wrong in principle or practice'

from nltk.corpus import wordnet as wn
   syn2 = wn.synset('good.a.01')
   antonym2 = syn2.lemmas()[0].antonyms()[0]
antonym2.name()

'bad'

antonym2.synset().definition()

'having undesirable or negative qualities’

import nltk

from nltk.stem import PorterStemmer

word_stemmer = PorterStemmer()

word_stemmer.stem('writing')

'write'

word_stemmer.stem('eating')

'eat'

import nltk
from nltk.stem import PorterStemmer
word_stemmer = PorterStemmer()
word_stemmer.stem('writing')

'write'

import nltk

from nltk.stem import LancasterStemmer

Lanc_stemmer = LancasterStemmer()

Lanc_stemmer.stem('eats')

'eat'

import nltk
from nltk.stem import LancatserStemmer
Lanc_stemmer = LancasterStemmer()
Lanc_stemmer.stem('eats')

'eat'

import nltk

from nltk.stem import RegexpStemmer

Reg_stemmer = RegexpStemmer(‘ing’)

Reg_stemmer.stem('eating')

'eat'

Reg_stemmer.stem('ingeat')

'eat'
Reg_stemmer.stem('eats')

'eat'

import nltk
from nltk.stem import RegexpStemmer
Reg_stemmer = RegexpStemmer()
Reg_stemmer.stem('ingeat')

'eat'

import nltk

from nltk.stem import SnowballStemmer

SnowballStemmer.languages

(
   'arabic',
   'danish',
   'dutch',
   'english',
   'finnish',
   'french',
   'german',
   'hungarian',
   'italian',
   'norwegian',
   'porter',
   'portuguese',
   'romanian',
   'russian',
   'spanish',
   'swedish'
)

French_stemmer = SnowballStemmer(‘french’)

French_stemmer.stem (‘Bonjoura’)

'bonjour'

import nltk
from nltk.stem import SnowballStemmer
French_stemmer = SnowballStemmer(‘french’)
French_stemmer.stem (‘Bonjoura’)

'bonjour'

import nltk

from nltk.stem import WordNetLemmatizer

lemmatizer = WordNetLemmatizer()

lemmatizer.lemmatize('eating')

'eating'

lemmatizer.lemmatize('books')

'book'

import nltk
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
lemmatizer.lemmatize('books')

'book'

believ

import nltk
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
lemmatizer.lemmatize(' believes ')

believ

import re
from nltk.corpus import wordnet

R_patterns = [
   (r'won\'t', 'will not'),
   (r'can\'t', 'cannot'),
   (r'i\'m', 'i am'),
   r'(\w+)\'ll', '\g<1> will'),
   (r'(\w+)n\'t', '\g<1> not'),
   (r'(\w+)\'ve', '\g<1> have'),
   (r'(\w+)\'s', '\g<1> is'),
   (r'(\w+)\'re', '\g<1> are'),
]

class REReplacer(object):
   def __init__(self, pattern = R_patterns):
      self.pattern = [(re.compile(regex), repl) for (regex, repl) in patterns]
   def replace(self, text):
      s = text
      for (pattern, repl) in self.pattern:
         s = re.sub(pattern, repl, s)
      return s

from repRE import REReplacer
rep_word = REReplacer()
rep_word.replace("I won't do it")
Output:
'I will not do it'
rep_word.replace("I can’t do it")
Output:
'I cannot do it'

import re
from nltk.corpus import wordnet
R_patterns = [
   (r'won\'t', 'will not'),
   (r'can\'t', 'cannot'),
   (r'i\'m', 'i am'),
   r'(\w+)\'ll', '\g<1> will'),
   (r'(\w+)n\'t', '\g<1> not'),
   (r'(\w+)\'ve', '\g<1> have'),
   (r'(\w+)\'s', '\g<1> is'),
   (r'(\w+)\'re', '\g<1> are'),
]
class REReplacer(object):
def __init__(self, patterns=R_patterns):
   self.patterns = [(re.compile(regex), repl) for (regex, repl) in patterns]
def replace(self, text):
   s = text
   for (pattern, repl) in self.patterns:
      s = re.sub(pattern, repl, s)
   return s

from replacerRE import REReplacer
rep_word = REReplacer()
rep_word.replace("I won't do it")

'I will not do it'

from nltk.tokenize import word_tokenize
from replacerRE import REReplacer
rep_word = REReplacer()
word_tokenize("I won't be able to do this now")
Output:
['I', 'wo', "n't", 'be', 'able', 'to', 'do', 'this', 'now']
word_tokenize(rep_word.replace("I won't be able to do this now"))
Output:
['I', 'will', 'not', 'be', 'able', 'to', 'do', 'this', 'now']

import re
from nltk.corpus import wordnet

class Rep_word_removal(object):
   def __init__(self):
      self.repeat_regexp = re.compile(r'(\w*)(\w)\2(\w*)')
      self.repl = r'\1\2\3'
   def replace(self, word):
      if wordnet.synsets(word):
      return word
   repl_word = self.repeat_regexp.sub(self.repl, word)
   if repl_word != word:
      return self.replace(repl_word)
   else:
      return repl_word

from removalrepeat import Rep_word_removal
rep_word = Rep_word_removal()
rep_word.replace ("Hiiiiiiiiiiiiiiiiiiiii")
Output:
'Hi'
rep_word.replace("Hellooooooooooooooo")
Output:
'Hello'

import re
from nltk.corpus import wordnet
class Rep_word_removal(object):
   def __init__(self):
      self.repeat_regexp = re.compile(r'(\w*)(\w)\2(\w*)')
      self.repl = r'\1\2\3'
   def replace(self, word):
      if wordnet.synsets(word):
         return word
   replace_word = self.repeat_regexp.sub(self.repl, word)
   if replace_word != word:
      return self.replace(replace_word)
   else:
      return replace_word

from removalrepeat import Rep_word_removal
rep_word = Rep_word_removal()
rep_word.replace ("Hiiiiiiiiiiiiiiiiiiiii")

'Hi'

import re
from nltk.corpus import wordnet

class word_syn_replacer(object):
   def __init__(self, word_map):
   self.word_map = word_map
def replace(self, word):
   return self.word_map.get(word, word)

from replacesyn import word_syn_replacer
rep_syn = word_syn_replacer ({‘bday’: ‘birthday’)
rep_syn.replace(‘bday’)

'birthday'

import re
from nltk.corpus import wordnet
class word_syn_replacer(object):
   def __init__(self, word_map):
   self.word_map = word_map
def replace(self, word):
   return self.word_map.get(word, word)

from replacesyn import word_syn_replacer
rep_syn = word_syn_replacer ({‘bday’: ‘birthday’)
rep_syn.replace(‘bday’)

'birthday'

import csv

class CSVword_syn_replacer(word_syn_replacer):
   def __init__(self, fname):
      word_map = {}
      for line in csv.reader(open(fname)):
         word, syn = line
         word_map[word] = syn
      super(Csvword_syn_replacer, self).__init__(word_map)

from replacesyn import CSVword_syn_replacer
rep_syn = CSVword_syn_replacer (‘syn.csv’)
rep_syn.replace(‘bday’)

'birthday'

import csv
class CSVword_syn_replacer(word_syn_replacer):
def __init__(self, fname):
word_map = {}
for line in csv.reader(open(fname)):
   word, syn = line
   word_map[word] = syn
super(Csvword_syn_replacer, self).__init__(word_map)

from replacesyn import CSVword_syn_replacer
rep_syn = CSVword_syn_replacer (‘syn.csv’)
rep_syn.replace(‘bday’)

'birthday'

import yaml

class YAMLword_syn_replacer(word_syn_replacer):
   def __init__(self, fname):
   word_map = yaml.load(open(fname))
   super(YamlWordReplacer, self).__init__(word_map)

from replacesyn import YAMLword_syn_replacer
rep_syn = YAMLword_syn_replacer (‘syn.yaml’)
rep_syn.replace(‘bday’)

'birthday'

import yaml
class YAMLword_syn_replacer(word_syn_replacer):
def __init__(self, fname):
   word_map = yaml.load(open(fname))
   super(YamlWordReplacer, self).__init__(word_map)

from replacesyn import YAMLword_syn_replacer
rep_syn = YAMLword_syn_replacer (‘syn.yaml’)
rep_syn.replace(‘bday’)

'birthday'

from nltk.corpus import wordnet

class word_antonym_replacer(object):
   def replace(self, word, pos=None):
      antonyms = set()
      for syn in wordnet.synsets(word, pos=pos):
         for lemma in syn.lemmas():
            for antonym in lemma.antonyms():
               antonyms.add(antonym.name())
      if len(antonyms) == 1:
         return antonyms.pop()
      else:
         return None
   def replace_negations(self, sent):
      i, l = 0, len(sent)
      words = []
      while i < l:
         word = sent[i]
         if word == 'not' and i+1 < l:
            ant = self.replace(sent[i+1])
            if ant:
               words.append(ant)
               i += 2
               continue
         words.append(word)
         i += 1
      return words

from replacerantonym import word_antonym_replacer
rep_antonym = word_antonym_replacer ()
rep_antonym.replace(‘uglify’)

['beautify'']
sentence = ["Let us", 'not', 'uglify', 'our', 'country']
rep_antonym.replace _negations(sentence)

["Let us", 'beautify', 'our', 'country']

nltk.corpus import wordnet
class word_antonym_replacer(object):
def replace(self, word, pos=None):
   antonyms = set()
   for syn in wordnet.synsets(word, pos=pos):
      for lemma in syn.lemmas():
      for antonym in lemma.antonyms():
         antonyms.add(antonym.name())
   if len(antonyms) == 1:
      return antonyms.pop()
   else:
      return None
def replace_negations(self, sent):
   i, l = 0, len(sent)
   words = []
   while i < l:
      word = sent[i]
      if word == 'not' and i+1 < l:
         ant = self.replace(sent[i+1])
         if ant:
            words.append(ant)
            i += 2
            continue
      words.append(word)
      i += 1
   return words

from replacerantonym import word_antonym_replacer
rep_antonym = word_antonym_replacer ()
rep_antonym.replace(‘uglify’)
sentence = ["Let us", 'not', 'uglify', 'our', 'country']
rep_antonym.replace _negations(sentence)

["Let us", 'beautify', 'our', 'country']

True

import nltk.data
path in nltk.data.path

True

import nltk.data
nltk.data.load(‘corpus/wordfile.txt’, format = ‘raw’)

b’tutorialspoint\n’

tutorialspoint
Online
Free
Tutorials

from nltk.corpus.reader import WordListCorpusReader
reader_corpus = WordListCorpusReader('.', ['list'])
reader_corpus.words()

['tutorialspoint', 'Online', 'Free', 'Tutorials']

The/at-tl expense/nn and/cc time/nn involved/vbn are/ber
astronomical/jj ./.

from nltk.corpus.reader import TaggedCorpusReader
reader_corpus = TaggedCorpusReader('.', r'.*\.pos')
reader_corpus.tagged_words()

[('The', 'AT-TL'), ('expense', 'NN'), ('and', 'CC'), ...]

[Earlier/JJR staff-reduction/NN moves/NNS] have/VBP trimmed/VBN about/
IN [300/CD jobs/NNS] ,/, [the/DT spokesman/NN] said/VBD ./.

from nltk.corpus.reader import ChunkedCorpusReader
reader_corpus = TaggedCorpusReader('.', r'.*\.chunk')
reader_corpus.chunked_words()

[
   Tree('NP', [('Earlier', 'JJR'), ('staff-reduction', 'NN'), ('moves', 'NNS')]),
   ('have', 'VBP'), ...
]

from nltk.corpus import brown^M
brown.categories()

[
   'adventure', 'belles_lettres', 'editorial', 'fiction', 'government',
   'hobbies', 'humor', 'learned', 'lore', 'mystery', 'news', 'religion',
   'reviews', 'romance', 'science_fiction'
]

from nltk.corpus.reader import CategorizedPlaintextCorpusReader
reader_corpus = CategorizedPlaintextCorpusReader('.', r'movie_.*\.txt',
cat_pattern = r'movie_(\w+)\.txt')
reader_corpus.categories()
reader_corpus.fileids(categories = [‘neg’])
reader_corpus.fileids(categories = [‘pos’])

['neg', 'pos']
['movie_neg.txt']
['movie_pos.txt']

import nltk
from nltk import word_tokenize
sentence = "I am going to school"
print (nltk.pos_tag(word_tokenize(sentence)))

[('I', 'PRP'), ('am', 'VBP'), ('going', 'VBG'), ('to', 'TO'), ('school', 'NN')]

import nltk
from nltk.tag import DefaultTagger
exptagger = DefaultTagger('NN')
exptagger.tag(['Tutorials','Point'])

[('Tutorials', 'NN'), ('Point', 'NN')]

import nltk
from nltk.tag import DefaultTagger
exptagger = DefaultTagger('NN')
from nltk.corpus import treebank
testsentences = treebank.tagged_sents() [1000:]
exptagger.evaluate (testsentences)

0.13198749536374715

import nltk
from nltk.tag import DefaultTagger
exptagger = DefaultTagger('NN')
exptagger.tag_sents([['Hi', ','], ['How', 'are', 'you', '?']])

[
   [
      ('Hi', 'NN'),
      (',', 'NN')
   ],
   [
      ('How', 'NN'),
      ('are', 'NN'),
      ('you', 'NN'),
      ('?', 'NN')
   ]
]

import nltk
from nltk.tag import untag
untag([('Tutorials', 'NN'), ('Point', 'NN')])

['Tutorials', 'Point']

from nltk.tag import UnigramTagger

from nltk.corpus import treebank

train_sentences = treebank.tagged_sents()[:2500]

Uni_tagger = UnigramTagger(train_sentences)

test_sentences = treebank.tagged_sents()[1500:]
Uni_tagger.evaluate(test_sents)

0.8942306156033808

from nltk.tag import UnigramTagger
from nltk.corpus import treebank
train_sentences = treebank.tagged_sents()[:2500]
Uni_tagger = UnigramTagger(train_sentences)
test_sentences = treebank.tagged_sents()[1500:]
Uni_tagger.evaluate(test_sentences)

0.8942306156033808

from nltk.tag import UnigramTagger
from nltk.corpus import treebank
Override_tagger = UnigramTagger(model = {‘Vinken’ : ‘NN’})
Override_tagger.tag(treebank.sents()[0])

[
   ('Pierre', None),
   ('Vinken', 'NN'),
   (',', None),
   ('61', None),
   ('years', None),
   ('old', None),
   (',', None),
   ('will', None),
   ('join', None),
   ('the', None),
   ('board', None),
   ('as', None),
   ('a', None),
   ('nonexecutive', None),
   ('director', None),
   ('Nov.', None),
   ('29', None),
   ('.', None)
]

from nltk.tag import UnigramTagger
from nltk.corpus import treebank
train_sentences = treebank.tagged_sents()[:2500]
Uni_tagger = UnigramTagger(train_sentences, cutoff = 4)
test_sentences = treebank.tagged_sents()[1500:]
Uni_tagger.evaluate(test_sentences)

0.7357651629613641

from nltk.tag import UnigramTagger
from nltk.tag import DefaultTagger
from nltk.corpus import treebank
train_sentences = treebank.tagged_sents()[:2500]
back_tagger = DefaultTagger('NN')
Uni_tagger = UnigramTagger(train_sentences, backoff = back_tagger)
test_sentences = treebank.tagged_sents()[1500:]
Uni_tagger.evaluate(test_sentences)

0.9061975746536931

import pickle
f = open('Uni_tagger.pickle','wb')
pickle.dump(Uni_tagger, f)
f.close()
f = open('Uni_tagger.pickle','rb')
Uni_tagger = pickle.load(f)

from nltk.tag import BigramTagger
from nltk.corpus import treebank
train_sentences = treebank.tagged_sents()[:2500]
Bi_tagger = BigramTagger(train_sentences)
test_sentences = treebank.tagged_sents()[1500:]
Bi_tagger.evaluate(test_sentences)

0.44669191071913594

from nltk.tag import TrigramTagger
from nltk.corpus import treebank
train_sentences = treebank.tagged_sents()[:2500]
Tri_tagger = TrigramTagger(train_sentences)
test_sentences = treebank.tagged_sents()[1500:]
Tri_tagger.evaluate(test_sentences)

0.41949863394526193

from tagger_util import backoff_tagger
from nltk.tag import UnigramTagger
from nltk.tag import BigramTagger
from nltk.tag import TrigramTagger
from nltk.tag import DefaultTagger
from nltk.corpus import treebank
train_sentences = treebank.tagged_sents()[:2500]
back_tagger = DefaultTagger('NN')
Combine_tagger = backoff_tagger(train_sentences,
[UnigramTagger, BigramTagger, TrigramTagger], backoff = back_tagger)
test_sentences = treebank.tagged_sents()[1500:]
Combine_tagger.evaluate(test_sentences)

0.9234530029238365

from nltk.tag import AffixTagger
from nltk.corpus import treebank
train_sentences = treebank.tagged_sents()[:2500]
Prefix_tagger = AffixTagger(train_sentences)
test_sentences = treebank.tagged_sents()[1500:]
Prefix_tagger.evaluate(test_sentences)

0.2800492099250667

from nltk.tag import AffixTagger
from nltk.corpus import treebank
train_sentences = treebank.tagged_sents()[:2500]
Prefix_tagger = AffixTagger(train_sentences, affix_length=4 )
test_sentences = treebank.tagged_sents()[1500:]
Prefix_tagger.evaluate(test_sentences)

0.18154947354966527

from nltk.tag import AffixTagger
from nltk.corpus import treebank
train_sentences = treebank.tagged_sents()[:2500]
Suffix_tagger = AffixTagger(train_sentences, affix_length = -3)
test_sentences = treebank.tagged_sents()[1500:]
Suffix_tagger.evaluate(test_sentences)

0.2800492099250667

templates = [
   brill.Template(brill.Pos([-1])),
   brill.Template(brill.Pos([1])),
   brill.Template(brill.Pos([-2])),
   brill.Template(brill.Pos([2])),
   brill.Template(brill.Pos([-2, -1])),
   brill.Template(brill.Pos([1, 2])),
   brill.Template(brill.Pos([-3, -2, -1])),
   brill.Template(brill.Pos([1, 2, 3])),
   brill.Template(brill.Pos([-1]), brill.Pos([1])),
   brill.Template(brill.Word([-1])),
   brill.Template(brill.Word([1])),
   brill.Template(brill.Word([-2])),
   brill.Template(brill.Word([2])),
   brill.Template(brill.Word([-2, -1])),
   brill.Template(brill.Word([1, 2])),
   brill.Template(brill.Word([-3, -2, -1])),
   brill.Template(brill.Word([1, 2, 3])),
   brill.Template(brill.Word([-1]), brill.Word([1])),
]
trainer = brill_trainer.BrillTaggerTrainer(initial_tagger, templates, deterministic=True)
return trainer.train(train_sentences, **kwargs)

from tagger_util import backoff_tagger
from nltk.tag import UnigramTagger
from nltk.tag import BigramTagger
from nltk.tag import TrigramTagger
from nltk.tag import DefaultTagger
from nltk.corpus import treebank
train_sentences = treebank.tagged_sents()[:2500]
back_tagger = DefaultTagger('NN')
Combine_tagger = backoff_tagger(
   train_sentences, [UnigramTagger, BigramTagger, TrigramTagger], backoff = back_tagger
)
test_sentences = treebank.tagged_sents()[1500:]
Combine_tagger.evaluate(test_sentences)

0.9234530029238365

from tagger_util import train_brill_tagger
brill_tagger = train_brill_tagger(combine_tagger, train_sentences)
brill_tagger.evaluate(test_sentences)

0.9246832510505041

from tagger_util import backoff_tagger
from nltk.tag import UnigramTagger
from nltk.tag import BigramTagger
from nltk.tag import TrigramTagger
from nltk.tag import DefaultTagger
from nltk.corpus import treebank
train_sentences = treebank.tagged_sents()[:2500]
back_tagger = DefaultTagger('NN')
Combine_tagger = backoff_tagger(train_sentences,
[UnigramTagger, BigramTagger, TrigramTagger], backoff = back_tagger)
test_sentences = treebank.tagged_sents()[1500:]
Combine_tagger.evaluate(test_sentences)
from tagger_util import train_brill_tagger
brill_tagger = train_brill_tagger(combine_tagger, train_sentences)
brill_tagger.evaluate(test_sentences)

0.9234530029238365
0.9246832510505041

from nltk.tag import tnt
from nltk.corpus import treebank
train_sentences = treebank.tagged_sents()[:2500]
tnt_tagger = tnt.TnT()
tnt_tagger.train(train_sentences)
test_sentences = treebank.tagged_sents()[1500:]
tnt_tagger.evaluate(test_sentences)

0.9165508316157791

[
   ((u'shot', u'VBD'), u'nsubj', (u'I', u'PRP')),
   ((u'shot', u'VBD'), u'dobj', (u'elephant', u'NN')),
   ((u'elephant', u'NN'), u'det', (u'an', u'DT')),
   ((u'shot', u'VBD'), u'prep', (u'in', u'IN')),
   ((u'in', u'IN'), u'pobj', (u'sleep', u'NN')),
   ((u'sleep', u'NN'), u'poss', (u'my', u'PRP$'))
]

import nltk
sentence = [
   ("the", "DT"),
   ("book", "NN"),
   ("has","VBZ"),
   ("many","JJ"),
   ("chapters","NNS")
]
chunker = nltk.RegexpParser(
   r'''
   NP:{<DT><NN.*><.*>*<NN.*>}
   }<VB.*>{
   '''
)
chunker.parse(sentence)
Output = chunker.parse(sentence)
Output.draw()

{<DT><NN>}

}<VB>{.

Import nltk
file = open (
   # provide here the absolute path for the file of text for which we want NER
)
data_text = file.read()
sentences = nltk.sent_tokenize(data_text)
tokenized_sentences = [nltk.word_tokenize(sentence) for sentence in sentences]
tagged_sentences = [nltk.pos_tag(sentence) for sentence in tokenized_sentences]
for sent in tagged_sentences:
print nltk.ne_chunk(sent)

import nltk
import re
IN = re.compile(r'.*\bin\b(?!\b.+ing)')
for doc in nltk.corpus.ieer.parsed_docs('NYT_19980315'):
for rel in nltk.sem.extract_rels('ORG', 'LOC', doc, corpus = 'ieer',
pattern = IN):
print(nltk.sem.rtuple(rel))

[ORG: 'WHYY'] 'in' [LOC: 'Philadelphia']
[ORG: 'McGlashan & Sarrail'] 'firm in' [LOC: 'San Mateo']
[ORG: 'Freedom Forum'] 'in' [LOC: 'Arlington']
[ORG: 'Brookings Institution'] ', the research group in' [LOC: 'Washington']
[ORG: 'Idealab'] ', a self-described business incubator based in' [LOC: 'Los Angeles']
[ORG: 'Open Text'] ', based in' [LOC: 'Waterloo']
[ORG: 'WGBH'] 'in' [LOC: 'Boston']
[ORG: 'Bastille Opera'] 'in' [LOC: 'Paris']
[ORG: 'Omnicom'] 'in' [LOC: 'New York']
[ORG: 'DDB Needham'] 'in' [LOC: 'New York']
[ORG: 'Kaplan Thaler Group'] 'in' [LOC: 'New York']
[ORG: 'BBDO South'] 'in' [LOC: 'Atlanta']
[ORG: 'Georgia-Pacific'] 'in' [LOC: 'Atlanta']

import nltk
def filter(chunk, tag_suffixes=['DT', 'CC']):
   significant = []
   for word, tag in chunk:
      ok = True
      for suffix in tag_suffixes:
         if tag.endswith(suffix):
            ok = False
            break
      if ok:
         significant.append((word, tag))
   return (significant)

from chunk_parse import filter
filter([('the', 'DT'),('good', 'JJ'),('movie', 'NN')])

[('good', 'JJ'), ('movie', 'NN')]

plural= {
   ('is', 'VBZ'): ('are', 'VBP'),
   ('was', 'VBD'): ('were', 'VBD')
}

singular = {
   ('are', 'VBP'): ('is', 'VBZ'),
   ('were', 'VBD'): ('was', 'VBD')
}

def index_chunk(chunk, pred, start = 0, step = 1):
   l = len(chunk)
   end = l if step > 0 else -1
   for i in range(start, end, step):
      if pred(chunk[i]):
         return i
      return None
def tag_startswith(prefix):
   def f(wt):
      return wt[1].startswith(prefix)
   return f

def verbs(chunk):
   vbidx = index_chunk(chunk, tag_startswith('VB'))
   if vbidx is None:
      return chunk
   verb, vbtag = chunk[vbidx]
   nnpred = tag_startswith('NN')
   nnidx = index_chunk(chunk, nnpred, start = vbidx+1)
   if nnidx is None:
      nnidx = index_chunk(chunk, nnpred, start = vbidx-1, step = -1)
   if nnidx is None:
      return chunk
   noun, nntag = chunk[nnidx]
   if nntag.endswith('S'):
      chunk[vbidx] = plural.get((verb, vbtag), (verb, vbtag))
   else:
      chunk[vbidx] = singular.get((verb, vbtag), (verb, vbtag))
   return chunk

from verbcorrect import verbs
verbs([('is', 'VBZ'), ('you', 'PRP$'), ('fine', 'VBG')])

[('are', 'VBP'), ('you', 'PRP$'), ('fine','VBG')]

def eliminate_passive(chunk):
   def vbpred(wt):
      word, tag = wt
      return tag != 'VBG' and tag.startswith('VB') and len(tag) > 2
   vbidx = index_chunk(chunk, vbpred)
   if vbidx is None:
      return chunk
   return chunk[vbidx+1:] + chunk[:vbidx]

from passiveverb import eliminate_passive
eliminate_passive(
   [
      ('the', 'DT'), ('tutorial', 'NN'), ('was', 'VBD'), ('great', 'JJ')
   ]
)

[('great', 'JJ'), ('the', 'DT'), ('tutorial', 'NN')]

def tag_eql(tag):
   def f(wt):
      return wt[1] == tag
   return f

def swapping_cardinals (chunk):
   cdidx = index_chunk(chunk, tag_eql('CD'))
   if not cdidx or not chunk[cdidx-1][1].startswith('NN'):
      return chunk
   noun, nntag = chunk[cdidx-1]
   chunk[cdidx-1] = chunk[cdidx]
   chunk[cdidx] = noun, nntag
   return chunk

from Cardinals import swapping_cardinals()
swapping_cardinals([('Janaury', 'NNP'), ('5', 'CD')])

[('10', 'CD'), ('January', 'NNP')]
10 January

from nltk.corpus import treebank_chunk
tree = treebank_chunk.chunked_sents()[2]
' '.join([w for w, t in tree.leaves()])

'Rudolph Agnew , 55 years old and former chairman of Consolidated Gold Fields
PLC , was named a nonexecutive director of this British industrial
conglomerate .'

from nltk.tree import Tree
def childtree_flat(trees):
   children = []
   for t in trees:
      if t.height() < 3:
         children.extend(t.pos())
      elif t.height() == 3:
         children.append(Tree(t.label(), t.pos()))
      else:
         children.extend(flatten_childtrees([c for c in t]))
   return children
def deeptree_flat(tree):
   return Tree(tree.label(), flatten_childtrees([c for c in tree]))

from deeptree import deeptree_flat
from nltk.corpus import treebank
deeptree_flat(treebank.parsed_sents()[2])

Tree('S', [Tree('NP', [('Rudolph', 'NNP'), ('Agnew', 'NNP')]),
(',', ','), Tree('NP', [('55', 'CD'),
('years', 'NNS')]), ('old', 'JJ'), ('and', 'CC'),
Tree('NP', [('former', 'JJ'),
('chairman', 'NN')]), ('of', 'IN'), Tree('NP', [('Consolidated', 'NNP'),
('Gold', 'NNP'), ('Fields', 'NNP'), ('PLC',
'NNP')]), (',', ','), ('was', 'VBD'),
('named', 'VBN'), Tree('NP-SBJ', [('*-1', '-NONE-')]),
Tree('NP', [('a', 'DT'), ('nonexecutive', 'JJ'), ('director', 'NN')]),
('of', 'IN'), Tree('NP',
[('this', 'DT'), ('British', 'JJ'),
('industrial', 'JJ'), ('conglomerate', 'NN')]), ('.', '.')])

from nltk.tree import Tree
def tree_shallow(tree):
   children = []
   for t in tree:
      if t.height() < 3:
         children.extend(t.pos())
      else:
         children.append(Tree(t.label(), t.pos()))
   return Tree(tree.label(), children)

from shallowtree import shallow_tree
from nltk.corpus import treebank
tree_shallow(treebank.parsed_sents()[2])

Tree('S', [Tree('NP-SBJ-1', [('Rudolph', 'NNP'), ('Agnew', 'NNP'), (',', ','),
('55', 'CD'), ('years', 'NNS'), ('old', 'JJ'), ('and', 'CC'),
('former', 'JJ'), ('chairman', 'NN'), ('of', 'IN'), ('Consolidated', 'NNP'),
('Gold', 'NNP'), ('Fields', 'NNP'), ('PLC', 'NNP'), (',', ',')]),
Tree('VP', [('was', 'VBD'), ('named', 'VBN'), ('*-1', '-NONE-'), ('a', 'DT'),
('nonexecutive', 'JJ'), ('director', 'NN'), ('of', 'IN'), ('this', 'DT'),
('British', 'JJ'), ('industrial', 'JJ'), ('conglomerate', 'NN')]), ('.', '.')])

from nltk.corpus import treebank
tree_shallow(treebank.parsed_sents()[2]).height()

3

from nltk.corpus import treebank
treebank.parsed_sents()[2].height()

9

from nltk.tree import Tree
def tree_convert(tree, mapping):
   children = []
   for t in tree:
      if isinstance(t, Tree):
         children.append(convert_tree_labels(t, mapping))
      else:
         children.append(t)
   label = mapping.get(tree.label(), tree.label())
   return Tree(label, children)

from converttree import tree_convert
from nltk.corpus import treebank
mapping = {'NP-SBJ': 'NP', 'NP-TMP': 'NP'}
convert_tree_labels(treebank.parsed_sents()[2], mapping)

Tree('S', [Tree('NP-SBJ-1', [Tree('NP', [Tree('NNP', ['Rudolph']),
Tree('NNP', ['Agnew'])]), Tree(',', [',']),
Tree('UCP', [Tree('ADJP', [Tree('NP', [Tree('CD', ['55']),
Tree('NNS', ['years'])]),
Tree('JJ', ['old'])]), Tree('CC', ['and']),
Tree('NP', [Tree('NP', [Tree('JJ', ['former']),
Tree('NN', ['chairman'])]), Tree('PP', [Tree('IN', ['of']),
Tree('NP', [Tree('NNP', ['Consolidated']),
Tree('NNP', ['Gold']), Tree('NNP', ['Fields']),
Tree('NNP', ['PLC'])])])])]), Tree(',', [','])]),
Tree('VP', [Tree('VBD', ['was']),Tree('VP', [Tree('VBN', ['named']),
Tree('S', [Tree('NP', [Tree('-NONE-', ['*-1'])]),
Tree('NP-PRD', [Tree('NP', [Tree('DT', ['a']),
Tree('JJ', ['nonexecutive']), Tree('NN', ['director'])]),
Tree('PP', [Tree('IN', ['of']), Tree('NP',
[Tree('DT', ['this']), Tree('JJ', ['British']), Tree('JJ', ['industrial']),
Tree('NN', ['conglomerate'])])])])])])]), Tree('.', ['.'])])

def bow(words):
   return dict([(word, True) for word in words])

from bagwords import bow
bow(['we', 'are', 'using', 'tutorialspoint'])

{'we': True, 'are': True, 'using': True, 'tutorialspoint': True}

def label_corpus(corp, feature_detector=bow):
   label_feats = collections.defaultdict(list)
   for label in corp.categories():
      for fileid in corp.fileids(categories=[label]):
         feats = feature_detector(corp.words(fileids=[fileid]))
         label_feats[label].append(feats)
   return label_feats

def split(lfeats, split=0.75):
   train_feats = []
   test_feats = []
   for label, feats in lfeats.items():
      cutoff = int(len(feats) * split)
      train_feats.extend([(feat, label) for feat in feats[:cutoff]])
      test_feats.extend([(feat, label) for feat in feats[cutoff:]])
   return train_feats, test_feats

from nltk.corpus import movie_reviews
from featx import label_feats_from_corpus, split_label_feats
movie_reviews.categories()

['neg', 'pos']

lfeats = label_feats_from_corpus(movie_reviews)
lfeats.keys()

dict_keys(['neg', 'pos'])

train_feats, test_feats = split_label_feats(lfeats, split = 0.75)
len(train_feats)

1500

len(test_feats)

500

from nltk.classify import NaiveBayesClassifier
NBC = NaiveBayesClassifier.train(train_feats)
NBC.labels()

['neg', 'pos']

from nltk.classify import DecisionTreeClassifier
decisiont_classifier = DecisionTreeClassifier.train(
   train_feats, binary = True, entropy_cutoff = 0.8,
   depth_cutoff = 5, support_cutoff = 30
)
accuracy(decisiont_classifier, test_feats)

0.725

from nltk.classify import MaxentClassifier
maxent_classifier = MaxentClassifier
.train(train_feats,algorithm = 'gis', trace = 0, max_iter = 10, min_lldelta = 0.5)
accuracy(maxent_classifier, test_feats)

0.786

0.885

import collections
from nltk import metrics
def metrics_PR(classifier, testfeats):
   refsets = collections.defaultdict(set)
   testsets = collections.defaultdict(set)
   for i, (feats, label) in enumerate(testfeats):
      refsets[label].add(i)
      observed = classifier.classify(feats)
         testsets[observed].add(i)
   precisions = {}
   recalls = {}
   for label in classifier.labels():
   precisions[label] = metrics.precision(refsets[label],testsets[label])
   recalls[label] = metrics.recall(refsets[label], testsets[label])
   return precisions, recalls

from metrics_classification import metrics_PR
nb_precisions, nb_recalls = metrics_PR(nb_classifier,test_feats)
nb_precisions['pos']

0.6713532466435213

nb_precisions['neg']

0.9676271186440678

nb_recalls['pos']

0.96

nb_recalls['neg']

0.478

['neg', 'pos']

accuracy(combined_classifier, test_feats)

0.948