$ python Python 3.9.9 | packaged by conda-forge | (main, Dec 20 2021, 02:38:53) [Clang 11.1.0 ] on darwin Type "help", "copyright", "credits" or "license" for more information. >>> text = 'Alice was beginning to get very tired of sitting by her sister on the bank, and of having nothing to do. Once or twice she had peeped into the book her sister was reading, but it had no pictures or conversations in it, "and what is the use of a book," thought Alice, "without pictures or conversations?"\nSo she was considering in her own mind (as well as she could, for the hot day made her feel very sleepy and stupid), whether the pleasure of making a daisy-chain would be worth the trouble of getting up and picking the daisies, when suddenly a White Rabbit with pink eyes ran close by her.' >>> len(text) 593 >>> text.split() ['Alice', 'was', 'beginning', 'to', 'get', 'very', 'tired', 'of', 'sitting', 'by', 'her', 'sister', 'on', 'the', 'bank,', 'and', 'of', 'having', 'nothing', 'to', 'do.', 'Once', 'or', 'twice', 'she', 'had', 'peeped', 'into', 'the', 'book', 'her', 'sister', 'was', 'reading,', 'but', 'it', 'had', 'no', 'pictures', 'or', 'conversations', 'in', 'it,', '"and', 'what', 'is', 'the', 'use', 'of', 'a', 'book,"', 'thought', 'Alice,', '"without', 'pictures', 'or', 'conversations?"', 'So', 'she', 'was', 'considering', 'in', 'her', 'own', 'mind', '(as', 'well', 'as', 'she', 'could,', 'for', 'the', 'hot', 'day', 'made', 'her', 'feel', 'very', 'sleepy', 'and', 'stupid),', 'whether', 'the', 'pleasure', 'of', 'making', 'a', 'daisy-chain', 'would', 'be', 'worth', 'the', 'trouble', 'of', 'getting', 'up', 'and', 'picking', 'the', 'daisies,', 'when', 'suddenly', 'a', 'White', 'Rabbit', 'with', 'pink', 'eyes', 'ran', 'close', 'by', 'her.'] >>> len(text.split()) 112 >>> >>> words = text.split() >>> len(words) 112 >>> len(text) / len(words) 5.294642857142857 >>> text 'Alice was beginning to get very tired of sitting by her sister on the bank, and of having nothing to do. Once or twice she had peeped into the book her sister was reading, but it had no pictures or conversations in it, "and what is the use of a book," thought Alice, "without pictures or conversations?"\nSo she was considering in her own mind (as well as she could, for the hot day made her feel very sleepy and stupid), whether the pleasure of making a daisy-chain would be worth the trouble of getting up and picking the daisies, when suddenly a White Rabbit with pink eyes ran close by her.' >>> words ['Alice', 'was', 'beginning', 'to', 'get', 'very', 'tired', 'of', 'sitting', 'by', 'her', 'sister', 'on', 'the', 'bank,', 'and', 'of', 'having', 'nothing', 'to', 'do.', 'Once', 'or', 'twice', 'she', 'had', 'peeped', 'into', 'the', 'book', 'her', 'sister', 'was', 'reading,', 'but', 'it', 'had', 'no', 'pictures', 'or', 'conversations', 'in', 'it,', '"and', 'what', 'is', 'the', 'use', 'of', 'a', 'book,"', 'thought', 'Alice,', '"without', 'pictures', 'or', 'conversations?"', 'So', 'she', 'was', 'considering', 'in', 'her', 'own', 'mind', '(as', 'well', 'as', 'she', 'could,', 'for', 'the', 'hot', 'day', 'made', 'her', 'feel', 'very', 'sleepy', 'and', 'stupid),', 'whether', 'the', 'pleasure', 'of', 'making', 'a', 'daisy-chain', 'would', 'be', 'worth', 'the', 'trouble', 'of', 'getting', 'up', 'and', 'picking', 'the', 'daisies,', 'when', 'suddenly', 'a', 'White', 'Rabbit', 'with', 'pink', 'eyes', 'ran', 'close', 'by', 'her.'] >>> >>> total = 0 >>> for word in words: ... total = total + len(word) ... >>> total 482 >>> total / len(words) 4.303571428571429 >>> words ['Alice', 'was', 'beginning', 'to', 'get', 'very', 'tired', 'of', 'sitting', 'by', 'her', 'sister', 'on', 'the', 'bank,', 'and', 'of', 'having', 'nothing', 'to', 'do.', 'Once', 'or', 'twice', 'she', 'had', 'peeped', 'into', 'the', 'book', 'her', 'sister', 'was', 'reading,', 'but', 'it', 'had', 'no', 'pictures', 'or', 'conversations', 'in', 'it,', '"and', 'what', 'is', 'the', 'use', 'of', 'a', 'book,"', 'thought', 'Alice,', '"without', 'pictures', 'or', 'conversations?"', 'So', 'she', 'was', 'considering', 'in', 'her', 'own', 'mind', '(as', 'well', 'as', 'she', 'could,', 'for', 'the', 'hot', 'day', 'made', 'her', 'feel', 'very', 'sleepy', 'and', 'stupid),', 'whether', 'the', 'pleasure', 'of', 'making', 'a', 'daisy-chain', 'would', 'be', 'worth', 'the', 'trouble', 'of', 'getting', 'up', 'and', 'picking', 'the', 'daisies,', 'when', 'suddenly', 'a', 'White', 'Rabbit', 'with', 'pink', 'eyes', 'ran', 'close', 'by', 'her.'] >>> [len(word) for word in words] [5, 3, 9, 2, 3, 4, 5, 2, 7, 2, 3, 6, 2, 3, 5, 3, 2, 6, 7, 2, 3, 4, 2, 5, 3, 3, 6, 4, 3, 4, 3, 6, 3, 8, 3, 2, 3, 2, 8, 2, 13, 2, 3, 4, 4, 2, 3, 3, 2, 1, 6, 7, 6, 8, 8, 2, 15, 2, 3, 3, 11, 2, 3, 3, 4, 3, 4, 2, 3, 6, 3, 3, 3, 3, 4, 3, 4, 4, 6, 3, 8, 7, 3, 8, 2, 6, 1, 11, 5, 2, 5, 3, 7, 2, 7, 2, 3, 7, 3, 8, 4, 8, 1, 5, 6, 4, 4, 4, 3, 5, 2, 4] >>> sum([len(word) for word in words]) >>> sum([len(word) for word in words]) 482 >>> sum([len(word) for word in words]) / len(words) 4.303571428571429 >>> import nltk >>> alice = nltk.corpus.gutenberg.words('carroll-alice.txt') >>> len(alice) 34110 >>> alice[:10] ['[', 'Alice', "'", 's', 'Adventures', 'in', 'Wonderland', 'by', 'Lewis', 'Carroll'] >>> alice[-10:] ['child', '-', 'life', ',', 'and', 'the', 'happy', 'summer', 'days', '.'] >>> alice[-1] '.' >>> total = 0 >>> for word in alice: ... total += len(word) ... >>> total 116009 >>> total / len(alice) 3.401026092055116 >>> >>> alice[:100] ['[', 'Alice', "'", 's', 'Adventures', 'in', 'Wonderland', 'by', 'Lewis', 'Carroll', '1865', ']', 'CHAPTER', 'I', '.', 'Down', 'the', 'Rabbit', '-', 'Hole', 'Alice', 'was', 'beginning', 'to', 'get', 'very', 'tired', 'of', 'sitting', 'by', 'her', 'sister', 'on', 'the', 'bank', ',', 'and', 'of', 'having', 'nothing', 'to', 'do', ':', 'once', 'or', 'twice', 'she', 'had', 'peeped', 'into', 'the', 'book', 'her', 'sister', 'was', 'reading', ',', 'but', 'it', 'had', 'no', 'pictures', 'or', 'conversations', 'in', 'it', ',', "'", 'and', 'what', 'is', 'the', 'use', 'of', 'a', 'book', ",'", 'thought', 'Alice', "'", 'without', 'pictures', 'or', 'conversation', "?'", 'So', 'she', 'was', 'considering', 'in', 'her', 'own', 'mind', '(', 'as', 'well', 'as', 'she', 'could', ','] >>> alice2 = [word for word in alice if word != ','] >>> len(alice2) 32117 >>> len(alice) 34110 >>> alice2 = [word for word in alice if word != ',' and word != '.'] >>> len(alice2) 31353 >>> alice2 = [word for word in alice if word != ',' and word != '.' and word != '?'] >>> len(alice2) 31318 >>> total / len(alice2) 3.704227600740788 >>> from statistics import mean >>> mean([len(word) for word in alice if word != ',' and word != '.' and word != '?']) >>> mean([len(word) for word in alice if word != ',' and word != '.' and word != '?']) 3.6150775911616324 >>> sum([len(word) for word in alice if word != ',' and word != '.' and word != '?']) / len(alice2) 3.6150775911616324 >>> fd = nltk.FreqDist([len(word) for word in alice]) >>> fd FreqDist({3: 7205, 1: 7093, 4: 5793, 2: 5647, 5: 3340, 6: 1952, 7: 1571, 8: 723, 9: 447, 10: 181, ...}) >>> fd.plot() >>> [word for word in alice if len(word) == 14] ['disappointment', 'Multiplication', 'contemptuously', 'contemptuously', 'affectionately'] >>> [word for word in alice if len(word) == 15] [] >>> [word for word in alice if len(word) == 13] ['conversations', 'inquisitively', 'uncomfortable', 'uncomfortable', 'circumstances', 'extraordinary', 'straightening', 'uncomfortable', 'extraordinary', 'uncomfortable', 'uncomfortably'] >>> vocab = set(alice) >>> >>> vocab = set(alice) >>> len(vocab) 3016 >>> len(alice) 34110 >>> mean([len(word) for word in vocab]) 5.812334217506631 >>> mean([len(word) for word in alice]) 3.401026092055116 >>> fd2 = nltk.FreqDist([len(word) for word in alice]) >>> fd2 = nltk.FreqDist([len(word) for word in vocab]) >>> fd1 = nltk.FreqDist([len(word) for word in alice]) >>> fd1 FreqDist({3: 7205, 1: 7093, 4: 5793, 2: 5647, 5: 3340, 6: 1952, 7: 1571, 8: 723, 9: 447, 10: 181, ...}) >>> fd2 FreqDist({4: 529, 5: 502, 6: 490, 7: 447, 8: 309, 3: 263, 9: 185, 2: 103, 10: 90, 11: 36, ...}) >>> fd2.plot() >>>