1. Programming For Big Data
Darren Redmond

2. Developing Out Spell Checking Capabilities
Based on the tutorial below – and we’ll see where we end up in class Create a file called spellcheck.py with the following: # to begin we start with a list of words in a file called spell.words # we read the file and strip out the file endings words = open('spell.words').readlines() words = map(lambda x: x.strip(), words) # now check if the word zygotic is a word print('zygotic' in words) python spellcheck.py

3. Create Functions For Reuse
# create a function to load the words and to check a word is in the dictionary def load_words(file_name): words = open(file_name).readlines() words = map(lambda x: x.strip(), words) return words def check_word(words, word): return word in words words = load_words('spell.words') # now check if the word zygotic is a word print(check_word(words, 'zygotic')) python spellcheck.py

4. Create Function To Check A Sentence
… def check_words(words, sentence): words_to_check = sentence.split(' ') for word in words_to_check: if not check_word(words, word): print('Word is misspelt : ' + word) return False return True words = load_words('spell.words') print(check_word(words, 'zygotic')) print(check_words(words, 'zygotic mistasdas elementary'))

5. The Full Program Now
def load_words(file_name): words = open(file_name).readlines() words = map(lambda x: x.strip(), words) return words def check_word(words, word): return word in words def check_words(words, sentence): words_to_check = sentence.split(' ') for word in words_to_check: if not check_word(words, word): print('Word is misspelt : ' + word) return False return True words = load_words('spell.words') print(check_word(words, 'zygotic')) print(check_words(words, 'zygotic mistasdas elementary'))

6. Create Spell Checker Class
class SpellChecker(object): def __init__(self): self.words = [] def load_words(self, file_name): self.words = open(file_name).readlines() self.words = map(lambda x: x.strip(), self.words) def check_word(self, word): return word in self.words def check_words(self, sentence): words_to_check = sentence.split(' ') for word in words_to_check: if not self.check_word(word): print('Word is misspelt : ' + word) return False return True

7. Use the Spell Checker Class
… # enable so that this is only called when the script run from the command line if __name__ == '__main__': spellChecker = SpellChecker() spellChecker.load_words('spell.words') print(spellChecker.check_word('zygotic')) print(spellChecker.check_words('zygotic mistasdas elementary'))

8. Let’s Create Some Unit Tests – spellcheck_test.py
import unittest from spellcheck import SpellChecker class TestSpellChecker(unittest.TestCase): def setUp(self): self.spellChecker = SpellChecker() self.spellChecker.load_words('spell.words') def test_spell_checker(self): self.assertTrue(self.spellChecker.check_word('zygotic')) self.assertFalse(self.spellChecker.check_words('zygotic mistasdas elementary')) self.assertTrue(self.spellChecker.check_words('our first correct sentence')) if __name__ == '__main__': unittest.main()

9. Let’s Find Some Bugs
import unittest from spellcheck import SpellChecker class TestSpellChecker(unittest.TestCase): def setUp(self): self.spellChecker = SpellChecker() self.spellChecker.load_words('spell.words') def test_spell_checker(self): self.assertTrue(self.spellChecker.check_word('zygotic')) self.assertFalse(self.spellChecker.check_words('zygotic mistasdas elementary')) self.assertTrue(self.spellChecker.check_words('our first correct sentence')) # handle case sensitivity self.assertTrue(self.spellChecker.check_words('Our first correct sentence')) # handle full stop self.assertTrue(self.spellChecker.check_words('Our first correct sentence.')) if __name__ == '__main__': unittest.main()

10. Let’s Fix Some Bugs
class SpellChecker(object): … def check_word(self, word): # remove full stops and ensure lower case – 2 bugs fixed. return word.strip('.').lower() in self.words

11. Handle Multiple Words Failing in a Sentence
import unittest from spellcheck import SpellChecker class TestSpellChecker(unittest.TestCase): def setUp(self): self.spellChecker = SpellChecker() self.spellChecker.load_words('spell.words') def test_spell_checker(self): self.assertTrue(self.spellChecker.check_word('zygotic')) self.assertFalse(self.spellChecker.check_words('zygotic mistasdas elementary')) self.assertTrue(self.spellChecker.check_words('our first correct sentence')) # handle case sensitivity self.assertTrue(self.spellChecker.check_words('Our first correct sentence')) # handle full stop self.assertTrue(self.spellChecker.check_words('Our first correct sentence.')) self.assertFalse(self.spellChecker.check_words('zygotic mistasdas spelllleeeing elementary')) …

12. Handle Multiple Failures in a Sentence Better
class SpellChecker(object): … def check_words(self, sentence): words_to_check = sentence.split(' ') failed_words = [] for word in words_to_check: if not self.check_word(word): print('Word is misspelt : ' + word) failed_words.append(word) return failed_words

13. Handle Multiple Failures in a Sentence Better
class TestSpellChecker(unittest.TestCase): … def test_spell_checker(self): self.assertTrue(self.spellChecker.check_word('zygotic')) failed_words = self.spellChecker.check_words('zygotic mistasdas elementary') self.assertEquals(1, len(failed_words)) self.assertEquals('mistasdas', failed_words[0]) self.assertEquals(0, len(self.spellChecker.check_words('our first correct sentence'))) # handle case sensitivity self.assertEquals(0, len(self.spellChecker.check_words('Our first correct sentence'))) # handle full stop self.assertEquals(0, len(self.spellChecker.check_words('Our first correct sentence.'))) failed_words = self.spellChecker.check_words('zygotic mistasdas spelllleeeing elementary') self.assertEquals(2, len(failed_words)) self.assertEquals('spelllleeeing', failed_words[1])

14. Handle a Document which is a list of Sentences
class SpellChecker(object): def load_file(self, file_name): lines = open(file_name).readlines() return map(lambda x: x.strip(), lines) def load_words(self, file_name): self.words = self.load_file(file_name) … def check_document(self, file_name): self.sentences = self.load_file(file_name) failed_words_in_sentences = [] index = 0 for sentence in self.sentences: failed_words_in_sentences.extend(self.check_words(sentence, index)) index = index + 1 return failed_words_in_sentences

15. Handle a Document which is a list of Sentences
class TestSpellChecker(unittest.TestCase): … def test_spell_checker(self): self.assertTrue(self.spellChecker.check_word('zygotic')) failed_words = self.spellChecker.check_words('zygotic mistasdas elementary') self.assertEquals(1, len(failed_words)) self.assertEquals('mistasdas', failed_words[0]) self.assertEquals(0, len(self.spellChecker.check_words('our first correct sentence'))) # handle case sensitivity self.assertEquals(0, len(self.spellChecker.check_words('Our first correct sentence'))) # handle full stop self.assertEquals(0, len(self.spellChecker.check_words('Our first correct sentence.'))) failed_words = self.spellChecker.check_words('zygotic mistasdas spelllleeeing elementary') self.assertEquals(2, len(failed_words)) self.assertEquals('spelllleeeing', failed_words[1]) # more bugs because the spell checker doesn’t spell check itself correctly – 21 entries not correct – dictionary words need to be lower self.assertEqual(21, len(self.spellChecker.check_document('spell.words')))

16. Handle lowering the case of the dictionary words
class SpellChecker(object): def load_file(self, file_name): lines = open(file_name).readlines() # ensures that all items read become lower case. return map(lambda x: x.strip().lower(), lines) … class TestSpellChecker(unittest.TestCase): def test_spell_checker(self): # fix the 21 issues of words not matching the dictionary self.assertEqual(0, len(self.spellChecker.check_document('spell.words')))

17. Tracking failed words, line number and caret position
In order to calculate the mis-spelt word, the line number, and the caret position – we will append a dict into the list of failed words instead of just the word. So: failed_words.append(word) Will become: failed_words.append({'word':word, 'line': line_number, 'pos': caret_position}) We will just need to keep track of the lines and the caret positions. So in the check_document function – we sohuld use the enumerate function to give us the index in the list: for sentence in self.sentences: failed_words_in_sentences.extend(self.check_words(sentence)) Becomes for index, sentence in enumerate(self.sentences): failed_words_in_sentences.extend(self.check_words(sentence, index))

18. Tracking failed words, line number and caret position
So the check_words function will become: def check_words(self, sentence, index): words_to_check = sentence.split(' ') caret_position = 0 failed_words = [] for word in words_to_check: if not self.check_word(word): print('Word is misspelt ' + word + ' at line : ' + str(index+1) + ' pos ' + str(caret_position+1)) failed_words.append({'word':word,'line':index+1,'pos':caret_position+1}) # update the caret position to be the length of the word plus 1 for the split character. caret_position = caret_position + len(word) + 1 return failed_words

19. Tracking failed words, line number and caret position
So the check_words function will become: # index = 0 is set here so that the function can be called for one line and index defaults to 0 def check_words(self, sentence, index = 0): words_to_check = sentence.split(' ') caret_position = 0 failed_words = [] for word in words_to_check: if not self.check_word(word): print('Word is misspelt ' + word + ' at line : ' + str(index+1) + ' pos ' + str(caret_position+1)) failed_words.append({'word':word,'line':index+1,'pos':caret_position+1}) # update the caret position to be the length of the word plus 1 for the split character. caret_position = caret_position + len(word) + 1 return failed_words

20. Updating the tests So the check_words function will become:
def test_spell_checker(self):
self.assertTrue(self.spellChecker.check_word('zygotic'))
failed_words = self.spellChecker.check_words('zygotic mistasdas elementary')
self.assertEquals(1, len(failed_words))
self.assertEquals('mistasdas', failed_words[0]['word'])
self.assertEquals(1, failed_words[0]['line'])
self.assertEquals(9, failed_words[0]['pos'])
self.assertEquals(0, len(self.spellChecker.check_words('our first correct sentence')))
# handle case sensitivity
self.assertEquals(0, len(self.spellChecker.check_words('Our capital sentence')))
# handle full stop
self.assertEquals(0, len(self.spellChecker.check_words('Our full stop sentence.')))
failed_words = self.spellChecker.check_words('zygotic mistasdas spelllleeeing elementary')
self.assertEquals(2, len(failed_words))
self.assertEquals('spelllleeeing', failed_words[1]['word'])
self.assertEquals(1, failed_words[1]['line'])
self.assertEquals(19, failed_words[1]['pos'])
self.assertEqual(0, len(self.spellChecker.check_document('spell.words')))

21. Look at the files spellcheck.py and spellcheck_test.py What next?
The Full Solution
Look at the files spellcheck.py and spellcheck_test.py
What next?
How to spell check on a directory of files? – hint os.listdir
How to handle a different languages? Dict with list per language.