Spaces:
Sleeping
Sleeping
| # Natural Language Toolkit: Product Reviews Corpus Reader | |
| # | |
| # Copyright (C) 2001-2023 NLTK Project | |
| # Author: Pierpaolo Pantone <[email protected]> | |
| # URL: <https://www.nltk.org/> | |
| # For license information, see LICENSE.TXT | |
| """ | |
| CorpusReader for reviews corpora (syntax based on Customer Review Corpus). | |
| Customer Review Corpus information | |
| ================================== | |
| Annotated by: Minqing Hu and Bing Liu, 2004. | |
| Department of Computer Science | |
| University of Illinois at Chicago | |
| Contact: Bing Liu, [email protected] | |
| https://www.cs.uic.edu/~liub | |
| Distributed with permission. | |
| The "product_reviews_1" and "product_reviews_2" datasets respectively contain | |
| annotated customer reviews of 5 and 9 products from amazon.com. | |
| Related papers: | |
| - Minqing Hu and Bing Liu. "Mining and summarizing customer reviews". | |
| Proceedings of the ACM SIGKDD International Conference on Knowledge | |
| Discovery & Data Mining (KDD-04), 2004. | |
| - Minqing Hu and Bing Liu. "Mining Opinion Features in Customer Reviews". | |
| Proceedings of Nineteeth National Conference on Artificial Intelligence | |
| (AAAI-2004), 2004. | |
| - Xiaowen Ding, Bing Liu and Philip S. Yu. "A Holistic Lexicon-Based Appraoch to | |
| Opinion Mining." Proceedings of First ACM International Conference on Web | |
| Search and Data Mining (WSDM-2008), Feb 11-12, 2008, Stanford University, | |
| Stanford, California, USA. | |
| Symbols used in the annotated reviews: | |
| :[t]: the title of the review: Each [t] tag starts a review. | |
| :xxxx[+|-n]: xxxx is a product feature. | |
| :[+n]: Positive opinion, n is the opinion strength: 3 strongest, and 1 weakest. | |
| Note that the strength is quite subjective. | |
| You may want ignore it, but only considering + and - | |
| :[-n]: Negative opinion | |
| :##: start of each sentence. Each line is a sentence. | |
| :[u]: feature not appeared in the sentence. | |
| :[p]: feature not appeared in the sentence. Pronoun resolution is needed. | |
| :[s]: suggestion or recommendation. | |
| :[cc]: comparison with a competing product from a different brand. | |
| :[cs]: comparison with a competing product from the same brand. | |
| Note: Some of the files (e.g. "ipod.txt", "Canon PowerShot SD500.txt") do not | |
| provide separation between different reviews. This is due to the fact that | |
| the dataset was specifically designed for aspect/feature-based sentiment | |
| analysis, for which sentence-level annotation is sufficient. For document- | |
| level classification and analysis, this peculiarity should be taken into | |
| consideration. | |
| """ | |
| import re | |
| from nltk.corpus.reader.api import * | |
| from nltk.tokenize import * | |
| TITLE = re.compile(r"^\[t\](.*)$") # [t] Title | |
| FEATURES = re.compile( | |
| r"((?:(?:\w+\s)+)?\w+)\[((?:\+|\-)\d)\]" | |
| ) # find 'feature' in feature[+3] | |
| NOTES = re.compile(r"\[(?!t)(p|u|s|cc|cs)\]") # find 'p' in camera[+2][p] | |
| SENT = re.compile(r"##(.*)$") # find tokenized sentence | |
| class Review: | |
| """ | |
| A Review is the main block of a ReviewsCorpusReader. | |
| """ | |
| def __init__(self, title=None, review_lines=None): | |
| """ | |
| :param title: the title of the review. | |
| :param review_lines: the list of the ReviewLines that belong to the Review. | |
| """ | |
| self.title = title | |
| if review_lines is None: | |
| self.review_lines = [] | |
| else: | |
| self.review_lines = review_lines | |
| def add_line(self, review_line): | |
| """ | |
| Add a line (ReviewLine) to the review. | |
| :param review_line: a ReviewLine instance that belongs to the Review. | |
| """ | |
| assert isinstance(review_line, ReviewLine) | |
| self.review_lines.append(review_line) | |
| def features(self): | |
| """ | |
| Return a list of features in the review. Each feature is a tuple made of | |
| the specific item feature and the opinion strength about that feature. | |
| :return: all features of the review as a list of tuples (feat, score). | |
| :rtype: list(tuple) | |
| """ | |
| features = [] | |
| for review_line in self.review_lines: | |
| features.extend(review_line.features) | |
| return features | |
| def sents(self): | |
| """ | |
| Return all tokenized sentences in the review. | |
| :return: all sentences of the review as lists of tokens. | |
| :rtype: list(list(str)) | |
| """ | |
| return [review_line.sent for review_line in self.review_lines] | |
| def __repr__(self): | |
| return 'Review(title="{}", review_lines={})'.format( | |
| self.title, self.review_lines | |
| ) | |
| class ReviewLine: | |
| """ | |
| A ReviewLine represents a sentence of the review, together with (optional) | |
| annotations of its features and notes about the reviewed item. | |
| """ | |
| def __init__(self, sent, features=None, notes=None): | |
| self.sent = sent | |
| if features is None: | |
| self.features = [] | |
| else: | |
| self.features = features | |
| if notes is None: | |
| self.notes = [] | |
| else: | |
| self.notes = notes | |
| def __repr__(self): | |
| return "ReviewLine(features={}, notes={}, sent={})".format( | |
| self.features, self.notes, self.sent | |
| ) | |
| class ReviewsCorpusReader(CorpusReader): | |
| """ | |
| Reader for the Customer Review Data dataset by Hu, Liu (2004). | |
| Note: we are not applying any sentence tokenization at the moment, just word | |
| tokenization. | |
| >>> from nltk.corpus import product_reviews_1 | |
| >>> camera_reviews = product_reviews_1.reviews('Canon_G3.txt') | |
| >>> review = camera_reviews[0] | |
| >>> review.sents()[0] # doctest: +NORMALIZE_WHITESPACE | |
| ['i', 'recently', 'purchased', 'the', 'canon', 'powershot', 'g3', 'and', 'am', | |
| 'extremely', 'satisfied', 'with', 'the', 'purchase', '.'] | |
| >>> review.features() # doctest: +NORMALIZE_WHITESPACE | |
| [('canon powershot g3', '+3'), ('use', '+2'), ('picture', '+2'), | |
| ('picture quality', '+1'), ('picture quality', '+1'), ('camera', '+2'), | |
| ('use', '+2'), ('feature', '+1'), ('picture quality', '+3'), ('use', '+1'), | |
| ('option', '+1')] | |
| We can also reach the same information directly from the stream: | |
| >>> product_reviews_1.features('Canon_G3.txt') | |
| [('canon powershot g3', '+3'), ('use', '+2'), ...] | |
| We can compute stats for specific product features: | |
| >>> n_reviews = len([(feat,score) for (feat,score) in product_reviews_1.features('Canon_G3.txt') if feat=='picture']) | |
| >>> tot = sum([int(score) for (feat,score) in product_reviews_1.features('Canon_G3.txt') if feat=='picture']) | |
| >>> mean = tot / n_reviews | |
| >>> print(n_reviews, tot, mean) | |
| 15 24 1.6 | |
| """ | |
| CorpusView = StreamBackedCorpusView | |
| def __init__( | |
| self, root, fileids, word_tokenizer=WordPunctTokenizer(), encoding="utf8" | |
| ): | |
| """ | |
| :param root: The root directory for the corpus. | |
| :param fileids: a list or regexp specifying the fileids in the corpus. | |
| :param word_tokenizer: a tokenizer for breaking sentences or paragraphs | |
| into words. Default: `WordPunctTokenizer` | |
| :param encoding: the encoding that should be used to read the corpus. | |
| """ | |
| CorpusReader.__init__(self, root, fileids, encoding) | |
| self._word_tokenizer = word_tokenizer | |
| self._readme = "README.txt" | |
| def features(self, fileids=None): | |
| """ | |
| Return a list of features. Each feature is a tuple made of the specific | |
| item feature and the opinion strength about that feature. | |
| :param fileids: a list or regexp specifying the ids of the files whose | |
| features have to be returned. | |
| :return: all features for the item(s) in the given file(s). | |
| :rtype: list(tuple) | |
| """ | |
| if fileids is None: | |
| fileids = self._fileids | |
| elif isinstance(fileids, str): | |
| fileids = [fileids] | |
| return concat( | |
| [ | |
| self.CorpusView(fileid, self._read_features, encoding=enc) | |
| for (fileid, enc) in self.abspaths(fileids, True) | |
| ] | |
| ) | |
| def reviews(self, fileids=None): | |
| """ | |
| Return all the reviews as a list of Review objects. If `fileids` is | |
| specified, return all the reviews from each of the specified files. | |
| :param fileids: a list or regexp specifying the ids of the files whose | |
| reviews have to be returned. | |
| :return: the given file(s) as a list of reviews. | |
| """ | |
| if fileids is None: | |
| fileids = self._fileids | |
| return concat( | |
| [ | |
| self.CorpusView(fileid, self._read_review_block, encoding=enc) | |
| for (fileid, enc) in self.abspaths(fileids, True) | |
| ] | |
| ) | |
| def sents(self, fileids=None): | |
| """ | |
| Return all sentences in the corpus or in the specified files. | |
| :param fileids: a list or regexp specifying the ids of the files whose | |
| sentences have to be returned. | |
| :return: the given file(s) as a list of sentences, each encoded as a | |
| list of word strings. | |
| :rtype: list(list(str)) | |
| """ | |
| return concat( | |
| [ | |
| self.CorpusView(path, self._read_sent_block, encoding=enc) | |
| for (path, enc, fileid) in self.abspaths(fileids, True, True) | |
| ] | |
| ) | |
| def words(self, fileids=None): | |
| """ | |
| Return all words and punctuation symbols in the corpus or in the specified | |
| files. | |
| :param fileids: a list or regexp specifying the ids of the files whose | |
| words have to be returned. | |
| :return: the given file(s) as a list of words and punctuation symbols. | |
| :rtype: list(str) | |
| """ | |
| return concat( | |
| [ | |
| self.CorpusView(path, self._read_word_block, encoding=enc) | |
| for (path, enc, fileid) in self.abspaths(fileids, True, True) | |
| ] | |
| ) | |
| def _read_features(self, stream): | |
| features = [] | |
| for i in range(20): | |
| line = stream.readline() | |
| if not line: | |
| return features | |
| features.extend(re.findall(FEATURES, line)) | |
| return features | |
| def _read_review_block(self, stream): | |
| while True: | |
| line = stream.readline() | |
| if not line: | |
| return [] # end of file. | |
| title_match = re.match(TITLE, line) | |
| if title_match: | |
| review = Review( | |
| title=title_match.group(1).strip() | |
| ) # We create a new review | |
| break | |
| # Scan until we find another line matching the regexp, or EOF. | |
| while True: | |
| oldpos = stream.tell() | |
| line = stream.readline() | |
| # End of file: | |
| if not line: | |
| return [review] | |
| # Start of a new review: backup to just before it starts, and | |
| # return the review we've already collected. | |
| if re.match(TITLE, line): | |
| stream.seek(oldpos) | |
| return [review] | |
| # Anything else is part of the review line. | |
| feats = re.findall(FEATURES, line) | |
| notes = re.findall(NOTES, line) | |
| sent = re.findall(SENT, line) | |
| if sent: | |
| sent = self._word_tokenizer.tokenize(sent[0]) | |
| review_line = ReviewLine(sent=sent, features=feats, notes=notes) | |
| review.add_line(review_line) | |
| def _read_sent_block(self, stream): | |
| sents = [] | |
| for review in self._read_review_block(stream): | |
| sents.extend([sent for sent in review.sents()]) | |
| return sents | |
| def _read_word_block(self, stream): | |
| words = [] | |
| for i in range(20): # Read 20 lines at a time. | |
| line = stream.readline() | |
| sent = re.findall(SENT, line) | |
| if sent: | |
| words.extend(self._word_tokenizer.tokenize(sent[0])) | |
| return words | |