diff --git a/NewsSentiment/controller_data/datasets/NewsMTSC-dataset/readme.md b/NewsSentiment/controller_data/datasets/NewsMTSC-dataset/readme.md index afba64f..c5ff2ad 100644 --- a/NewsSentiment/controller_data/datasets/NewsMTSC-dataset/readme.md +++ b/NewsSentiment/controller_data/datasets/NewsMTSC-dataset/readme.md @@ -9,7 +9,7 @@ The dataset consists of three splits. In practical terms, we suggest to use the * `train.jsonl` - For **training**. * `devtest_mt.jsonl` - To evaluate a model's classification performance only on sentences that contain **at least two target mentions**. Note that the mentions were extracted to refer to different persons but in a few cases might indeed refer to the same person since we extracted them automatically. -* `devtest_mt.jsonl` - To evaluate a model's classification performance on a "**real-world**" set of sentences, i.e., the set was created with the objective to resemble real-world distribution as to sentiment and other factors mentioned in the paper. +* `devtest_rw.jsonl` - To evaluate a model's classification performance on a "**real-world**" set of sentences, i.e., the set was created with the objective to resemble real-world distribution as to sentiment and other factors mentioned in the paper. ### Format diff --git a/NewsSentiment/dataset.py b/NewsSentiment/dataset.py index 250adc2..821124c 100644 --- a/NewsSentiment/dataset.py +++ b/NewsSentiment/dataset.py @@ -1,6 +1,18 @@ import random from collections import Counter -from typing import List, Iterable, Set +from typing import ( + List, + Iterable, + Set, + Union, + Dict, + overload, + Sequence, + Mapping, + Literal, + Tuple, + Optional, +) import jsonlines import networkx as nx @@ -16,6 +28,7 @@ XLNetTokenizer, AlbertTokenizer, ) +from math import ceil from NewsSentiment.SentimentClasses import SentimentClasses from NewsSentiment.customexceptions import TooLongTextException, TargetNotFoundException @@ -109,13 +122,33 @@ def _get_labels(cls): parser_index = cls.NLP.pipe_names.index("parser") cls.NLP_DEP_PARSER_LABELS = list(cls.NLP.pipeline[parser_index][1].labels) + @staticmethod + @overload + def create_entire_text( + text_left: str, + target_phrase: str, + text_right: str, + is_return_modified_text_left: Literal[True], + ) -> Tuple[str, str]: + ... + + @staticmethod + @overload + def create_entire_text( + text_left: str, + target_phrase: str, + text_right: str, + is_return_modified_text_left: Literal[False], + ) -> str: + ... + @staticmethod def create_entire_text( text_left: str, target_phrase: str, text_right: str, is_return_modified_text_left: bool, - ): + ) -> Union[Tuple[str, str], str]: """ Create the entire text from the three prepared text segment. Will modify text_left (append a whitespace " ") if it is empty, which is necessary for @@ -152,8 +185,7 @@ def prepare_left_segment(text_left: str): :param text_left: :return: """ - while len(text_left) > 0 and text_left[0] == "\n": - text_left = text_left[1:] + text_left = text_left.lstrip("\n") if len(text_left) > 0 and text_left[-1] != " ": text_left += " " return text_left @@ -165,9 +197,7 @@ def prepare_target_mention(target_mention: str): :param target_mention: :return: """ - while len(target_mention) > 0 and target_mention[0] == "\n": - target_mention = target_mention[1:] - return target_mention + return target_mention.lstrip("\n") @staticmethod def prepare_right_segment(text_right: str): @@ -183,53 +213,82 @@ def prepare_right_segment(text_right: str): def _create_word_to_wordpiece_mapping( self, tokenizer, words: List, for_text_with_special_tokens, offset + ): + batch_output = self._batch_create_word_to_wordpiece_mapping( + tokenizer, [words], for_text_with_special_tokens, [offset] + ) + return tuple(output[0] for output in batch_output) + + def _batch_create_word_to_wordpiece_mapping( + self, + tokenizer, + words_per_target: List[List], + for_text_with_special_tokens, + offsets, ): if not for_text_with_special_tokens: raise NotImplementedError() - previous_words = [] - mapping = None - # offset word_index by 1 (0th word has index 1) - # words_without_single_whitespace = [word for word in words if word != " "] - for word_index, word in enumerate(words, start=offset): - # update left and previous words - left = " ".join(previous_words) + " " - previous_words.append(word) - - # produce target masks - target_mask = self._create_target_mask( - tokenizer=tokenizer, - text_left=left, - target=word, - text_right="", - for_text_with_special_tokens=for_text_with_special_tokens, - is_raise_exception_if_target_after_max_seq_len=False, + target_words = [ + ( + " ".join(target[:word_index]) + " ", # left + word, # target + "", # right ) + for target in words_per_target + for word_index, word in enumerate(target) + ] + + # produce target masks + target_masks = self._batch_create_target_mask( + tokenizer=tokenizer, + targets=target_words, + for_text_with_special_tokens=for_text_with_special_tokens, + is_raise_exception_if_target_after_max_seq_len=False, + ) - # if the current word is after the max seq len, the target mask = "toolong" - if target_mask == "toolong": - # we abort the for loop since all remaining words will also be - # after the max seq len - break + # split back into targets + _indexer = 0 + target_masks_per_target = [ + target_masks[_indexer : (_indexer := _indexer + len(wpt))] + for wpt in words_per_target + ] + + # remove "toolong" (word is after the max seq len) + valid_target_masks_per_target = [] + valid_tokens = [] + for target, tokens in zip(target_masks_per_target, words_per_target): + indexer = target.index("toolong") if "toolong" in target else None + valid_target_masks_per_target.append(target[:indexer]) + valid_tokens.append(tokens[:indexer]) + + # convert to tensors + mappings = [] + + for target, offset in zip(valid_target_masks_per_target, offsets): + target_tensor = torch.LongTensor(target) + + word_indices_multiplier = torch.tensor( + range(offset, offset + len(target_tensor)) + ) + target_tensor = target_tensor.mul(word_indices_multiplier.unsqueeze(-1)) - target_mask = torch.LongTensor(target_mask) - target_mask = torch.mul(target_mask, word_index) - if mapping is None: - mapping = target_mask + if type(tokenizer) == RobertaTokenizer: + if target_tensor.prod(0).sum(0) > 0: + logger.debug( + "overlap when mapping tokens to wordpiece (allow overwriting because" + " Roberta is used)" + ) else: - # make sure there is no overlap - if type(tokenizer) == RobertaTokenizer: - if sum(mapping * target_mask) > 0: - logger.debug( - "overlap when mapping tokens to wordpiece (allow overwriting because " - "Roberta is used)" - ) - else: - assert sum(mapping * target_mask) == 0 - mapping = mapping + target_mask + assert target_tensor.prod(0).sum(0) == 0 + + target_tensor = target_tensor.sum(0) + + assert target_tensor.size(0) <= self.max_seq_len - assert mapping.shape[0] <= self.max_seq_len - return mapping, previous_words + mappings.append(target_tensor) + + return mappings, valid_tokens def _calculate_dep_matrix(self, text_tokens, text_tokens_as_str): _check_len_doc = len(text_tokens) @@ -380,24 +439,76 @@ def _create_target_mask( :param text_pair: if not None, will be appended to the input sequence as the second pair :return: """ + return self._batch_create_target_mask( + tokenizer, + ((text_left, target, text_right),), + for_text_with_special_tokens, + is_raise_exception_if_target_after_max_seq_len, + )[0] + + def _batch_create_target_mask( + self, + tokenizer, + targets, + for_text_with_special_tokens: bool, + is_raise_exception_if_target_after_max_seq_len: bool = True, + ): if not for_text_with_special_tokens: raise NotImplementedError() - assert target != " ", "passed a single whitespace as target" - text, text_left = self.create_entire_text( - text_left, target, text_right, is_return_modified_text_left=True + lefts = [] + rights = [] + full_texts = [] + target_phrases = [] + for target in targets: + ft, l = self.create_entire_text(*target, is_return_modified_text_left=True) + lefts.append(l) + full_texts.append(ft) + target_phrases.append(target[1]) + rights.append(target[2]) + + encodings = self._encode_for_target_mask( + tokenizer, lefts, target_phrases, rights, full_texts ) + return [ + self._create_target_mask_on_encoding( + tokenizer, *encoding, is_raise_exception_if_target_after_max_seq_len + ) + for encoding in zip(*encodings) + ] + + def _encode_for_target_mask( + self, tokenizer, text_lefts, targets, text_rights, full_texts + ): # get token ids, cf. https://huggingface.co/transformers/glossary.html - text_left_ids_with_special_tokens = tokenizer.encode( - text_left, add_special_tokens=True - ) - text_right_ids = tokenizer.encode(text_right, add_special_tokens=False) - text_ids_with_special_tokens = tokenizer.encode(text, add_special_tokens=True) - target_phrase_ids_with_special_tokens = tokenizer.encode( - target, add_special_tokens=True + text_left_ids_with_special_tokens = tokenizer( + text_lefts, add_special_tokens=True + )["input_ids"] + text_right_ids = tokenizer(text_rights, add_special_tokens=False)["input_ids"] + text_ids_with_special_tokens = tokenizer(full_texts, add_special_tokens=True)[ + "input_ids" + ] + target_phrase_ids_with_special_tokens = tokenizer( + targets, add_special_tokens=True + )["input_ids"] + + return ( + text_left_ids_with_special_tokens, + target_phrase_ids_with_special_tokens, + text_right_ids, + text_ids_with_special_tokens, ) + def _create_target_mask_on_encoding( + self, + tokenizer, + text_left_ids_with_special_tokens, + target_phrase_ids_with_special_tokens, + text_right_ids, + text_ids_with_special_tokens, + is_raise_exception_if_target_after_max_seq_len, + ): len_text_left_ids_with_special_tokens = len(text_left_ids_with_special_tokens) len_text_right_ids = len(text_right_ids) len_text_id_with_special_tokens = len(text_ids_with_special_tokens) @@ -516,8 +627,8 @@ def _create_knowledge_source_tensor( self.max_seq_len, num_categories, dtype=torch.long ) for word_index, word in enumerate(text_tokens_as_str, start=offset): - positions_of_current_word_in_input_sequence = self._map_token_index_to_wordpiece_index( - mapping, word_index + positions_of_current_word_in_input_sequence = ( + self._map_token_index_to_wordpiece_index(mapping, word_index) ) if mode == "nrc_emotions": emotion_tensor = get_nrc_emotions_as_tensor(word) @@ -539,28 +650,40 @@ def _create_knowledge_source_tensor( return emotions_for_sequence def _create_mapping_from_tokenbased_to_wordpiece_based(self, text, tok_obj): + batch_output = self._batch_create_mapping_from_tokenbased_to_wordpiece_based( + (text,), tok_obj + ) + return tuple(output[0] for output in batch_output) + + def _batch_create_mapping_from_tokenbased_to_wordpiece_based(self, texts, tok_obj): offset = 10 # for spacy, we need to remove leading spaces as they will yield a single # token - text_without_leading_space = text.strip() - nlp_text = self.NLP(text_without_leading_space) + text_without_leading_space = [text.strip() for text in texts] + nlp_text_per_target = self.NLP.pipe(text_without_leading_space) # get only non-single-space tokens, see # https://github.com/explosion/spaCy/issues/1707 - text_tokens = [token for token in nlp_text if not token.is_space] - text_tokens_as_str = [token.text for token in nlp_text if not token.is_space] + text_tokens, text_tokens_as_str = zip( + *( + zip(*((token, token.text) for token in nlp_text if not token.is_space)) + for nlp_text in nlp_text_per_target + ) + ) # in case there is a word at position k that is longer than the max seq len # when converted to wordpiece indexes, _create_word_to_wordpiece_mapping # returns all words up to including k-1 - mapping, text_tokens_as_str = self._create_word_to_wordpiece_mapping( + mapping, text_tokens_as_str = self._batch_create_word_to_wordpiece_mapping( tok_obj, text_tokens_as_str, for_text_with_special_tokens=True, - offset=offset, + offsets=[offset] * len(texts), ) # ..., correspondingly, truncate text_tokens to same length - text_tokens = text_tokens[: len(text_tokens_as_str)] - return mapping, offset, text_tokens_as_str, text_tokens + text_tokens = [ + tt[: len(ttas)] for tt, ttas in zip(text_tokens, text_tokens_as_str) + ] + return mapping, [offset] * len(texts), text_tokens_as_str, text_tokens def _convert_non_null_to_one(self, lst): lst_one = [] @@ -600,7 +723,10 @@ def _create_dependency_tensor( dependency_column = dependency_tensor_of_tokens[word_index].tolist() dependency_column_ones = self._convert_non_null_to_one(dependency_column) _check_num_heads = sum(dependency_column_ones) - assert _check_num_heads in [0, 1,], ( + assert _check_num_heads in [ + 0, + 1, + ], ( f"expected only zero or one heads, found {_check_num_heads} for word " f"{word} in {text_tokens_as_str}" ) @@ -617,11 +743,15 @@ def _create_dependency_tensor( dependency_type = dependency_column[head_index] assert dependency_type != 0 - positions_of_current_word_in_input_sequence = self._map_token_index_to_wordpiece_index( - mapping_token2wordpiece, word_index_withoffset + positions_of_current_word_in_input_sequence = ( + self._map_token_index_to_wordpiece_index( + mapping_token2wordpiece, word_index_withoffset + ) ) - positions_of_current_head_in_input_sequence = self._map_token_index_to_wordpiece_index( - mapping_token2wordpiece, head_index_withoffset + positions_of_current_head_in_input_sequence = ( + self._map_token_index_to_wordpiece_index( + mapping_token2wordpiece, head_index_withoffset + ) ) for position_word in positions_of_current_word_in_input_sequence: @@ -669,8 +799,10 @@ def _create_dependency_tree_hop_distances_of_tokens_to_target( ): word_index_without_offset = word_index - mapping_token2wordpiece_offset - positions_of_current_word_in_input_sequence = self._map_token_index_to_wordpiece_index( - mapping_token2wordpiece, word_index + positions_of_current_word_in_input_sequence = ( + self._map_token_index_to_wordpiece_index( + mapping_token2wordpiece, word_index + ) ) depdistance_of_current_word_in_input_sequence = dist[ @@ -686,103 +818,189 @@ def _create_dependency_tree_hop_distances_of_tokens_to_target( def create_model_input_seqs( self, - text_left, - target_phrase, - text_right, - coreferential_targets_for_target_mask: Iterable[dict], - ): + text_left: str, + target_phrase: str, + text_right: str, + coreferential_targets_for_target_mask: Optional[Iterable[dict]], + ) -> Mapping[str, Mapping[str, Union[torch.Tensor, bool]]]: """ Creates input sequences for a given target. Prior components have processed the jsonl so that coreferential_targets_for_target_mask will contain a list of coreferential mentions of the target if and only if coref_mode == "in_targetmask". In that case, will produce an individual target mask for each coref mention and merge them to the target mask of the preferred mention. - :param text_left: - :param target_phrase: - :param text_right: - :param further_mentions: - :return: """ - text = self.create_entire_text( - text_left, target_phrase, text_right, is_return_modified_text_left=False + batch_result = self.batch_create_model_input_seqs( + ((text_left, target_phrase, text_right),), + coreferential_targets_for_target_mask=( + coreferential_targets_for_target_mask, + ), ) - logger.debug(f"'{text_left}' '{target_phrase}' '{text_right}'") + return { + tok_name: { + seq_name: seq_result[0] for seq_name, seq_result in tok_result.items() + } + for tok_name, tok_result in batch_result.items() + } + + @overload + def batch_create_model_input_seqs( + self, + targets: ..., + coreferential_targets_for_target_mask: ..., + ) -> Mapping[str, Mapping[str, Union[torch.Tensor, Tuple[bool], bool]]]: + ... - input_seqs_per_tokenizer = {} + @overload + def batch_create_model_input_seqs( + self, + targets: ..., + coreferential_targets_for_target_mask: ..., + ) -> Mapping[str, Mapping[str, Union[torch.Tensor, bool]]]: + ... - for name, tok_obj in self.tokenizers_name_and_obj.items(): - logger.debug(f"{name}") + def batch_create_model_input_seqs( + self, + targets: Sequence[Tuple[str, str, str]], + coreferential_targets_for_target_mask: Optional[ + Sequence[Optional[Iterable[dict]]] + ], + ) -> Mapping[str, Mapping[str, Union[torch.Tensor, Tuple[bool], bool]]]: + """ + Creates input sequences for given targets. Prior components have processed the jsonl + so that coreferential_targets_for_target_mask will contain a list of coreferential + mentions of the targets if and only if coref_mode == "in_targetmask". In that case, + will produce an individual target mask for each coref mention and merge them to + the target mask of the preferred mention. + + NEW + Multiple target inputs. Output will be a tuple of batches with the + created sequences being of shape [batch_size,seq_length]. To output a single + target without batch output and with tensors of shape [seq_length], + set single_target_output = True. + """ + num_targets = len(targets) + + if not coreferential_targets_for_target_mask: + coreferential_targets_for_target_mask = [None] * num_targets + elif num_targets != len(coreferential_targets_for_target_mask): + raise TypeError( + "Number of coreferential_targets_for_target_mask must match number of targets." + ) + + lefts = [] + target_phrases = [] + adjusted_target_phrases = [] + full_texts = [] + + for target in targets: + assert all(isinstance(arg, str) for arg in target), "Wrong input types." + + left, target_phrase, _ = target + + lefts.append(left) + + target_phrases.append(target_phrase) + + adjusted_target_phrases.append( + f"What do you think of {target_phrase}?" + if self.is_use_natural_target_phrase_for_spc + else target_phrase + ) + + full_texts.append( + self.create_entire_text(*target, is_return_modified_text_left=False) + ) + + out: Mapping[str, Mapping[str, Union[torch.Tensor, bool]]] = {} + + for tok_name, tok_obj in self.tokenizers_name_and_obj.items(): + logger.debug(f"{tok_name}") # text - text_ids_with_special_tokens = tok_obj.encode( - text, + text_ids_with_special_tokens_per_target = tok_obj( + text=full_texts, max_length=self.max_seq_len, - pad_to_max_length=True, + padding="max_length", add_special_tokens=True, truncation="longest_first", + )["input_ids"] + text_then_target_ids_with_special_tokens_dict_per_target = ( + tok_obj.batch_encode_plus( + zip(full_texts, adjusted_target_phrases), + max_length=self.max_seq_len, + padding="max_length", + add_special_tokens=True, + truncation="longest_first", + ) ) - # text SEP target - adjusted_target_phrase = target_phrase - if self.is_use_natural_target_phrase_for_spc: - adjusted_target_phrase = "What do you think of " + target_phrase + "?" - text_then_target_ids_with_special_tokens_dict = tok_obj.encode_plus( - text, - text_pair=adjusted_target_phrase, - max_length=self.max_seq_len, - pad_to_max_length=True, - add_special_tokens=True, - truncation="longest_first", + text_then_target_ids_with_special_tokens_dict_per_target = ( + text_then_target_ids_with_special_tokens_dict_per_target.data ) - text_then_target_ids_with_special_tokens_dict = ( - text_then_target_ids_with_special_tokens_dict.data + text_then_target_ids_with_special_tokens_per_target = ( + text_then_target_ids_with_special_tokens_dict_per_target["input_ids"] ) - text_then_target_ids_with_special_tokens = text_then_target_ids_with_special_tokens_dict[ - "input_ids" - ] if type(tok_obj) == RobertaTokenizer: - # roberta doesnt have segments, so we produce fake segment ids here, and later simply dont pass them - text_then_target_ids_with_special_tokens_segment_ids = [0] * len( - text_then_target_ids_with_special_tokens - ) - else: - text_then_target_ids_with_special_tokens_segment_ids = text_then_target_ids_with_special_tokens_dict[ - "token_type_ids" + # roberta doesnt have segments, so we produce fake segment ids + # here, and later simply dont pass them + text_then_target_ids_with_special_tokens_segment_ids_per_target = [ + [0 for _ in target_ids] + for target_ids in text_then_target_ids_with_special_tokens_per_target ] + else: + text_then_target_ids_with_special_tokens_segment_ids_per_target = ( + text_then_target_ids_with_special_tokens_dict_per_target[ + "token_type_ids" + ] + ) # target - target_ids_with_special_tokens = tok_obj.encode( - target_phrase, + target_ids_with_special_tokens_per_target = tok_obj( + target_phrases, max_length=self.max_seq_len, - pad_to_max_length=True, + padding="max_length", add_special_tokens=True, truncation="longest_first", - ) + )["input_ids"] # create target masks - target_mask_seq_for_text_with_special_tokens = self._create_target_mask( - tok_obj, - text_left, - target_phrase, - text_right, - for_text_with_special_tokens=True, + target_mask_seq_for_text_with_special_tokens_per_target = ( + self._batch_create_target_mask( + tok_obj, targets, for_text_with_special_tokens=True + ) ) # create target mask for coreferential targets of given target - coref_target_masks = self._create_coreferential_target_masks( - tok_obj, coreferential_targets_for_target_mask + coref_target_masks_per_target = ( + self._batch_create_coreferential_target_masks( + tok_obj, coreferential_targets_for_target_mask + ) ) # merge them into the preferred target mask - merged_target_mask = self._merge_coref_target_masks_into_preferred_target_mask( - target_mask_seq_for_text_with_special_tokens, coref_target_masks + merged_target_mask_per_target = tuple( + self._merge_coref_target_masks_into_preferred_target_mask( + target_mask, + coref_target_masks, + ) + for target_mask, coref_target_masks in zip( + target_mask_seq_for_text_with_special_tokens_per_target, + coref_target_masks_per_target, + ) + ) + target_mask_seq_for_text_with_special_tokens_per_target = ( + merged_target_mask_per_target ) - target_mask_seq_for_text_with_special_tokens = merged_target_mask # create also text ids without max length to see if the one that will be # used was truncated - text_ids_with_special_tokens_no_max_length = tok_obj.encode( - text, add_special_tokens=True + text_ids_with_special_tokens_no_max_length_per_target = tok_obj( + full_texts, add_special_tokens=True + )["input_ids"] + text_num_truncated_tokens_per_target = tuple( + len(noml) - len(ml) + for noml, ml in zip( + text_ids_with_special_tokens_no_max_length_per_target, + text_ids_with_special_tokens_per_target, + ) ) - text_num_truncated_tokens = len( - text_ids_with_special_tokens_no_max_length - ) - len(text_ids_with_special_tokens) # iterate tokens of full text and look up in dicts # create mapping from token-based indexes to wordpiece-based indexes @@ -795,100 +1013,143 @@ def create_model_input_seqs( # tokens anymore, because this list of now cut off. thus, i changed the assert in # _calculate_dep_distance to throwing an exception that is catched later on ( + mapping_token2wordpiece_per_target, + mapping_token2wordpiece_offset_per_target, + text_tokens_as_str_per_target, + text_tokens_per_target, + ) = self._batch_create_mapping_from_tokenbased_to_wordpiece_based( + full_texts, tok_obj + ) + + text_stacked_knowledge_source_info_per_target = [] + text_dependency_tree_hop_distances_per_target = [] + text_dependency_matrix_per_target = [] + + for ( + target_phrase, + left, mapping_token2wordpiece, mapping_token2wordpiece_offset, text_tokens_as_str, text_tokens, - ) = self._create_mapping_from_tokenbased_to_wordpiece_based(text, tok_obj) - - # create additional knowledge source tensors - # stack only those knowledge sources that were requested by arguments - selected_tensors_knowledge_sources_text = [] - for source in self.knowledge_sources: - text_tensor = self._create_knowledge_source_tensor( - text_tokens_as_str, - mapping_token2wordpiece_offset, - mapping_token2wordpiece, - source, - ) - selected_tensors_knowledge_sources_text.append(text_tensor) - if self.knowledge_sources: - text_stacked_knowledge_source_info = torch.cat( - tuple(selected_tensors_knowledge_sources_text), dim=1 + ) in zip( + target_phrases, + lefts, + mapping_token2wordpiece_per_target, + mapping_token2wordpiece_offset_per_target, + text_tokens_as_str_per_target, + text_tokens_per_target, + ): + # create additional knowledge source tensors + # stack only those knowledge sources that were requested by arguments + selected_tensors_knowledge_sources_text = [] + for source in self.knowledge_sources: + text_tensor = self._create_knowledge_source_tensor( + text_tokens_as_str, + mapping_token2wordpiece_offset, + mapping_token2wordpiece, + source, + ) + selected_tensors_knowledge_sources_text.append(text_tensor) + text_stacked_knowledge_source_info_per_target.append( + torch.cat(tuple(selected_tensors_knowledge_sources_text), dim=1) + if self.knowledge_sources + else None ) - else: - text_stacked_knowledge_source_info = None - # syntax hop distance - text_dependency_tree_hop_distances = self._create_dependency_tree_hop_distances_of_tokens_to_target( - text_tokens, - len(text_left), - target_phrase, - text_tokens_as_str, - mapping_token2wordpiece, - mapping_token2wordpiece_offset, - ) + # syntax hop distance + text_dependency_tree_hop_distances_per_target.append( + self._create_dependency_tree_hop_distances_of_tokens_to_target( + text_tokens, + len(left), + target_phrase, + text_tokens_as_str, + mapping_token2wordpiece, + mapping_token2wordpiece_offset, + ) + ) - text_dependency_matrix = self._create_dependency_tensor( - text_tokens, - text_tokens_as_str, - mapping_token2wordpiece, - mapping_token2wordpiece_offset, - ) + text_dependency_matrix_per_target.append( + self._create_dependency_tensor( + text_tokens, + text_tokens_as_str, + mapping_token2wordpiece, + mapping_token2wordpiece_offset, + ) + ) # create item with indexes and masks - input_seqs_per_tokenizer[name] = { + result = { FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( - text_ids_with_special_tokens + text_ids_with_special_tokens_per_target ), FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS_TARGET_MASK: torch.FloatTensor( - target_mask_seq_for_text_with_special_tokens + target_mask_seq_for_text_with_special_tokens_per_target + ), + FIELD_IS_OVERFLOW: tuple( + text_num > 0 for text_num in text_num_truncated_tokens_per_target ), - FIELD_IS_OVERFLOW: text_num_truncated_tokens > 0, FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( - text_then_target_ids_with_special_tokens + text_then_target_ids_with_special_tokens_per_target ), - # we used to have text-then-target target mask here, but won't use it, - # since it would be identical to the text target mask (since we only - # want to mark the target within the text, but not in the 2nd target - # component) - # FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS_TARGET_MASK: torch.LongTensor( - # text_then_target_ids_with_special_tokens_target_mask, - # ), FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS_SEGMENT_IDS: torch.LongTensor( - text_then_target_ids_with_special_tokens_segment_ids + text_then_target_ids_with_special_tokens_segment_ids_per_target ), FIELD_TARGET_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( - target_ids_with_special_tokens + target_ids_with_special_tokens_per_target + ), + FIELD_SYNTAX_HOP_DISTANCE_TO_TARGET: torch.stack( + text_dependency_tree_hop_distances_per_target + ), + FIELD_SYNTAX_DEPENDENCY_MATRIX: torch.stack( + text_dependency_matrix_per_target ), - FIELD_SYNTAX_HOP_DISTANCE_TO_TARGET: text_dependency_tree_hop_distances, - FIELD_SYNTAX_DEPENDENCY_MATRIX: text_dependency_matrix, } # add knowledge source if requested if self.knowledge_sources: - input_seqs_per_tokenizer[name][ + result[ FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS_SELECTED_KNOWLEDGE_SOURCES - ] = text_stacked_knowledge_source_info - # likewise as for FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS_TARGET_MASK, we - # don't need to set a special knowledge source mask for text-then-target + ] = torch.stack(text_stacked_knowledge_source_info_per_target) - return input_seqs_per_tokenizer + # add tokenizer result to output + out[tok_name] = result + + return out def _create_coreferential_target_masks( - self, tok_obj, coreferential_targets_for_target_mask + self, tok_obj, coreferential_targets_for_target_mask: Optional[Iterable[dict]] ): - target_masks = [] - for coref_target in coreferential_targets_for_target_mask: - coref_target_mask = self._create_target_mask( - tok_obj, - self.prepare_left_segment(coref_target["text_left"]), - self.prepare_target_mention(coref_target["mention"]), - "", - for_text_with_special_tokens=True, - ) - target_masks.append(coref_target_mask) - return target_masks + return self._batch_create_coreferential_target_masks( + tok_obj, (coreferential_targets_for_target_mask,) + )[0] + + def _batch_create_coreferential_target_masks( + self, + tok_obj, + coreferential_targets_for_target_mask_per_target: Sequence[ + Optional[Iterable[dict]] + ], + ): + target_masks_per_target = [] + for coreferential_targets in coreferential_targets_for_target_mask_per_target: + target_mask = [] + if coreferential_targets: + coref_targets = [ + ( + self.prepare_left_segment(coref_target["text_left"]), + self.prepare_target_mention(coref_target["mention"]), + "", + ) + for coref_target in coreferential_targets + ] + target_mask.append( + self._batch_create_target_mask( + tok_obj, coref_targets, for_text_with_special_tokens=True + ) + ) + target_masks_per_target.append(target_mask) + return target_masks_per_target def _merge_coref_target_masks_into_preferred_target_mask( self, target_mask_seq_for_text_with_special_tokens, coref_target_masks @@ -1110,7 +1371,10 @@ def _create_target_inputs( target_mention = FXEasyTokenizer.prepare_target_mention(target_mention) text_right = FXEasyTokenizer.prepare_right_segment(text_right) text = FXEasyTokenizer.create_entire_text( - text_left, target_mention, text_right, is_return_modified_text_left=False, + text_left, + target_mention, + text_right, + is_return_modified_text_left=False, ) # text to indexes @@ -1393,8 +1657,10 @@ def task_to_dataset_item(self, task, coref_mode: str, ignore_parsing_errors): # if the coref mode (only during training though) is additional_examples, repeat them here if coref_mode == "additional_examples": - expanded_targets = self._expand_coref_mentions_of_targets_to_multiple_targets( - targets, text + expanded_targets = ( + self._expand_coref_mentions_of_targets_to_multiple_targets( + targets, text + ) ) elif coref_mode == "in_targetmask": expanded_targets = targets @@ -1420,8 +1686,10 @@ def task_to_dataset_item(self, task, coref_mode: str, ignore_parsing_errors): example_id, text, expanded_targets ) else: - items = self._convert_multi_targets_in_single_item_to_k_targets_in_multi_items( - example_id, text, expanded_targets + items = ( + self._convert_multi_targets_in_single_item_to_k_targets_in_multi_items( + example_id, text, expanded_targets + ) ) # iterate (virtual) items of this actual row in the jsonl and create model items diff --git a/NewsSentiment/infer.py b/NewsSentiment/infer.py index 10f4316..db71d23 100644 --- a/NewsSentiment/infer.py +++ b/NewsSentiment/infer.py @@ -2,8 +2,10 @@ import torch import torch.nn.functional as F +from math import ceil from jsonlines import jsonlines from tqdm import tqdm +from typing import overload, Any, List, Dict, Union, Sequence, Tuple, Optional from NewsSentiment.SentimentClasses import SentimentClasses from NewsSentiment.dataset import FXEasyTokenizer @@ -83,75 +85,225 @@ def infer_from_text(self, left, target, right): """ return self.infer(text_left=left, target_mention=target, text_right=right) + @overload def infer( self, - text_left: str = None, - target_mention: str = None, - text_right: str = None, - text: str = None, - target_mention_from: int = None, - target_mention_to: int = None, - ): - """ - Calculates sentiment as to target_mention in a text that is a concatenation of - text_left, - target_mention, and text_right. Note that text_left and text_right should end - with a space (or comma, etc.)), - or end with a space, respectively. Alternatively, the target can be selected via - target_mention_from and target_mention_to in text. + text_left: str = ..., + target_mention: str = ..., + text_right: str = ..., + text: None = None, + target_mention_from: None = None, + target_mention_to: None = None, + targets: None = None, + batch_size: int = ..., + disable_tqdm: bool = ..., + ) -> Tuple[Dict[str, Any], ...]: + ... + + @overload + def infer( + self, + text_left: None = None, + target_mention: None = None, + text_right: None = None, + text: str = ..., + target_mention_from: int = ..., + target_mention_to: int = ..., + targets: None = None, + batch_size: int = ..., + disable_tqdm: bool = ..., + ) -> Tuple[Dict[str, Any], ...]: + ... + + @overload + def infer( + self, + text_left: None = None, + target_mention: None = None, + text_right: None = None, + text: None = None, + target_mention_from: None = None, + target_mention_to: None = None, + targets: Sequence[Union[Tuple[str, str, str], Tuple[str, int, int]]] = ..., + batch_size: int = ..., + disable_tqdm: bool = ..., + ) -> List[Tuple[Dict[str, Any], ...]]: + ... + + def infer( + self, + text_left: Optional[str] = None, + target_mention: Optional[str] = None, + text_right: Optional[str] = None, + text: Optional[str] = None, + target_mention_from: Optional[int] = None, + target_mention_to: Optional[int] = None, + targets: Optional[ + Sequence[Union[Tuple[str, str, str], Tuple[str, int, int]]] + ] = None, + batch_size: int = 1, + disable_tqdm: bool = False, + ) -> Union[Tuple[Dict[str, Any], ...], List[Tuple[Dict[str, Any], ...]]]: + """Computes sentiment for one or more targets. Additionally splits multiple + targets into batches of batch_size and processes them in a vectorized fashion. + + Note that the text before and after the target should end with a space + (or comma, etc.)), or begin with a space, respectively. + + Args: + text_left (str | None, optional): Text before the target mention. + Defaults to None. + target_mention (str | None, optional): Target mention. Defaults to None. + text_right (str | None, optional): Text after the target mention. + Defaults to None. + text (str | None, optional): Text containing the target mention. + Defaults to None. + target_mention_from (str | None, optional): Start index of the + target mention. Defaults to None. + target_mention_to (str | None, optional): End index of the target mention. + Defaults to None. + targets (Sequence[Tuple[str, str, str] | Tuple[str, int, int]] | None, + optional): Tuples containing (text_left,target_mention,text_right) + or (text,target_mention_from,target_mention_to) for multiple targets + (mixed style is possible). Defaults to None. + batch_size (int, optional): Preferred size of each batch if using multiple + targets. Defaults to 1. + disable_tqdm (bool, optional): Disables the tqdm progress bar that shows + progress in batch processing. Defaults to False. + + Returns: + Tuple[Dict[str, Any], ...] | List[Tuple[Dict[str, Any], ...]: Tuple (or + list of tuples for multiple targets with order preserved) containing + class probabilities as dictionaries with keys "class_id", "class_label" + and "class_prob". """ - is_index_based = ( - text is not None - and target_mention_from is not None - and target_mention_to is not None - ) - is_component_based = ( - text_left is not None - and target_mention is not None - and text_right is not None + component_base = (text_left, target_mention, text_right) + is_component_based = all(isinstance(i, str) for i in component_base) + + index_base = (text, target_mention_from, target_mention_to) + is_index_based = all( + isinstance(arg, typ) for arg, typ in zip(index_base, (str, int, int)) ) - assert is_index_based != is_component_based - - if text: - text_left = text[:target_mention_from] - target_mention = text[target_mention_from:target_mention_to] - text_right = text[target_mention_from:] - - # assert text_left.endswith(' ') # we cannot handle commas, if we have this - # check - assert not target_mention.startswith(" ") and not target_mention.endswith( - " " - ), f"target_mention={target_mention}; text={text}" - # assert text_right.startswith(' ') - - text_left = FXEasyTokenizer.prepare_left_segment(text_left) - target_mention = FXEasyTokenizer.prepare_target_mention(target_mention) - text_right = FXEasyTokenizer.prepare_right_segment(text_right) - - indexed_example = self.tokenizer.create_model_input_seqs( - text_left, target_mention, text_right, [] + + is_targets_based = targets is not None + + # verify input + assert ( + sum((is_component_based, is_index_based, is_targets_based)) == 1 + ), """Wrong input types or too many inputs! + Must be either one single or multiple component or index based targets.""" + + targets = ( + (component_base if is_component_based else index_base,) + if not is_targets_based + else targets ) - inputs = self.instructor.select_inputs(indexed_example, is_single_item=True) - # invoke model - outputs = self.model(inputs) - class_probabilites = F.softmax(outputs, dim=-1).reshape((3,)).cpu().tolist() - - classification_result = [] - for class_id, class_prob in enumerate(class_probabilites): - classification_result.append( - { - "class_id": class_id, - "class_label": self.polarities_inverse[class_id], - "class_prob": class_prob, - } + num_targets = len(targets) + num_batches = ceil(num_targets / batch_size) + + out = [] + + for batch_start, batch_end in tqdm( + zip( + range(0, num_targets, batch_size), + (*range(batch_size, num_targets, batch_size), None), + ), + total=num_batches, + desc="Processing batches", + unit="batch", + disable=disable_tqdm, + ): + out.extend(self.batch_infer(targets[batch_start:batch_end])) + + return out[0] if not is_targets_based else out + + def batch_infer( + self, + targets: Sequence[Union[Tuple[str, str, str], Tuple[str, int, int]]], + ) -> List[Tuple[Dict[str, Any], ...]]: + """Computes sentiment for a batch of targets. + Targets are tuples of text before target, target mention, and text after target. + + Args: + targets (Sequence[Tuple[str,str,str] | Tuple[str,int,int]]): Batch of + targets to compute sentiment for. Tuples contain + (text before, target mention, text after) or + (text,target mention start index, target mention end index). + Texts before and after the target mention should end with a space + (or comma, etc.)), or begin with a space, respectively. + + Returns: + List[Tuple[Dict[str,Any], ...]]: List of target classification tuples, + containing class probabilities as dictionaries with keys + "class_id", "class_label" and "class_prob". + The order of tuples matches the order of input targets. + """ + targets_prepared = [] + + for target in targets: + assert ( + len(target) == 3 + ), f"{target} is missing {3-len(target)} component(s)." + + if all(isinstance(component, str) for component in target): + text_left, target_mention, text_right = target + elif all( + isinstance(component, typ) + for component, typ in zip(target, (str, int, int)) + ): + text = target[0] + target_mention_from = target[1] + target_mention_to = target[2] + text_left = text[:target_mention_from] + target_mention = text[target_mention_from:target_mention_to] + text_right = text[target_mention_to:] + else: + raise TypeError(f"Wrong input types in {target}.") + + # assert text_left.endswith(' ') # we cannot handle commas, if we have this + # check + assert not any( + (target_mention.startswith(" "), target_mention.endswith(" ")) + ), f"target_mention={target_mention}; text={text}" + # assert text_right.startswith(' ') + + targets_prepared.append( + ( + FXEasyTokenizer.prepare_left_segment(text_left), + FXEasyTokenizer.prepare_target_mention(target_mention), + FXEasyTokenizer.prepare_right_segment(text_right), + ) ) - classification_result = sorted( - classification_result, key=lambda x: x["class_prob"], reverse=True + indexed_examples = self.tokenizer.batch_create_model_input_seqs( + targets=targets_prepared, + coreferential_targets_for_target_mask=None, ) - return classification_result + inputs = self.instructor.select_inputs(indexed_examples, is_single_item=False) + + outputs = self.model(inputs) + + class_probabilities_per_target = F.softmax(outputs, dim=-1).cpu().tolist() + + return [ + tuple( + sorted( + ( + { + "class_id": class_id, + "class_label": self.polarities_inverse[class_id], + "class_prob": class_prob, + } + for class_id, class_prob in enumerate(class_probabilities) + ), + key=lambda x: x["class_prob"], + reverse=True, + ) + ) + for class_probabilities in class_probabilities_per_target + ] def get_info_for_label(self, classification_result, label): for r in classification_result: @@ -183,7 +335,9 @@ def parse_arguments(override_args=False): "CPU", ) parser.add_argument( - "--logging", type=str, default="ERROR", + "--logging", + type=str, + default="ERROR", ) # if own_args == None -> parse_args will use sys.argv diff --git a/NewsSentiment/train.py b/NewsSentiment/train.py index db5629e..0c7532b 100644 --- a/NewsSentiment/train.py +++ b/NewsSentiment/train.py @@ -6,6 +6,7 @@ import time from collections import Counter from typing import Iterable +import logging import numpy import torch @@ -261,9 +262,19 @@ def create_transformer_model( self.transformer_tokenizers[ pretrained_weights_name ] = tokenizer_class.from_pretrained(model_path) + + # supress the transformers warning + # "Some weights of the model checkpoint..were not used.." + transformers_logger = logging.getLogger('transformers.modeling_utils') + transformers_logger_level = transformers_logger.getEffectiveLevel() + transformers_logger.setLevel(logging.ERROR) + self.transformer_models[ pretrained_weights_name ] = model_class.from_pretrained(model_path, output_hidden_states=True) + + # reset transformers logging level + transformers_logger.setLevel(transformers_logger_level) def _reset_params_of_own_model(self): for child in self.own_model.children(): @@ -379,7 +390,7 @@ def _train(self, criterion, optimizer, train_data_loader, dev_data_loader): for i_batch, sample_batched in enumerate(train_data_loader): global_step += 1 # clear gradient accumulators - optimizer.zero_grad() + optimizer.zero_grad(set_to_none = False) # select only relevant fields inputs = self.select_inputs(sample_batched) targets = sample_batched["polarity"].to(self.opt.device) diff --git a/README.md b/README.md index 6aff4af..d712e27 100644 --- a/README.md +++ b/README.md @@ -25,21 +25,23 @@ NewsMTSC was tested on MacOS and Ubuntu; other OS may work, too. Let us know :-) **1. Setup the environment:** -This step is optional if you have Python 3.7 installed already (`python --version`). If you don't have Python 3.7, we recommend using Anaconda for setting up requirements. If you do not have it yet, follow Anaconda's +This step is optional if you have Python 3.8 installed already (`python --version`). If you don't have Python 3.8, we recommend using Anaconda for setting up requirements. If you do not have it yet, follow Anaconda's [installation instructions](https://docs.anaconda.com/anaconda/install/). -To setup a Python 3.7 environment (in case you don't have one yet) you may use, for example: +To setup a Python 3.8 environment (in case you don't have one yet) you may use, for example: ```bash -conda create --yes -n newsmtsc python=3.7 +conda create --yes -n newsmtsc python=3.8 conda activate newsmtsc ``` FYI, for users of virtualenv, the equivalent command would be: ```bash -virtualenv -ppython3.7 --setuptools 45 venv +virtualenv -ppython3.8 --setuptools 45 venv source venv/bin/activate ``` +Note: We recommend Python 3.8, however we have successfully tested NewsMTSC with Python version >=3.8, <3.12. + **2. Setup NewsMTSC:** ```bash git clone git@github.com:fhamborg/NewsMTSC.git diff --git a/READMEpypi.md b/READMEpypi.md index befcc33..220de59 100644 --- a/READMEpypi.md +++ b/READMEpypi.md @@ -16,7 +16,7 @@ dataset, the model, and its source code can be viewed in our [GitHub repository] # Installation It's super easy, we promise! -You just need a Python 3.7 or Python 3.8 environment. See [here](https://raw.githubusercontent.com/fhamborg/NewsMTSC/main/pythoninfo.md) if you +You just need a Python 3.8 environment. See [here](https://raw.githubusercontent.com/fhamborg/NewsMTSC/main/pythoninfo.md) if you don't have Python or a different version (run `python --version` in a terminal to see your version). Then run: @@ -37,13 +37,36 @@ Since this is a one-time process, future use of NewsSentiment will be much faste from NewsSentiment import TargetSentimentClassifier tsc = TargetSentimentClassifier() -sentiment = tsc.infer_from_text("I like " ,"Peter", " but I don't like Robert.") -print(sentiment[0]) +data = [ + ("I like ", "Peter", " but I don't like Robert."), + ("", "Mark Meadows", "'s coverup of Trump’s coup attempt is falling apart."), +] + +sentiments = tsc.infer(targets=data) + +for i, result in enumerate(sentiments): + print("Sentiment: ", i, result[0]) +``` + +This method will internally split the data into batches of size 16 for increased speed. You can adjust the +batch size using the `batch_size` parameter, e.g., `batch_size=32`. + +Alternatively, you can also use the `infer_from_text` method to infer sentiment for a single target: -sentiment = tsc.infer_from_text("" ,"Mark Meadows", "'s coverup of Trump’s coup attempt is falling apart.") +```python +sentiment = tsc.infer_from_text("I like " ,"Peter", " but I don't like Robert.") print(sentiment[0]) ``` +# How to identify a person in a sentence? + +In case your data is not separated as shown in the examples above, i.e., in three segments, you will need to identify one (or more) targets first. +How this is done best depends on your project and analysis task but you may, for example, use NER. This [example](https://github.com/fhamborg/NewsMTSC/issues/30#issuecomment-1700645679) shows a simple way of doing so. + +# Acknowledgements + +Thanks to [Tilman Hornung](https://github.com/t1h0) for adding the batching functionality and various other improvements. + # How to cite If you use the dataset or model, please cite our [paper](https://www.aclweb.org/anthology/2021.eacl-main.142/) ([PDF](https://www.aclweb.org/anthology/2021.eacl-main.142.pdf)): diff --git a/pythoninfo.md b/pythoninfo.md index 757afe5..2052c39 100644 --- a/pythoninfo.md +++ b/pythoninfo.md @@ -1,13 +1,14 @@ -This step is optional if you have Python 3.7 or 3.8 installed (run `python --version` -in a terminal and check the version that is printed). If you don't have Python 3.7, we -recommend using Anaconda for setting up requirements because it is very easy (but any way -of installing Python 3.7 is fine). If you do not have Anaconda yet, follow their +This step is optional if you have Python >=3.8, <3.12 installed (run `python --version` +in a terminal and check the version that is printed; we recommend 3.8). If you don't +have Python (in the correct version), we recommend using Anaconda for setting up +requirements because it is very easy (but any way of installing is fine). +If you do not have Anaconda yet, follow their [installation instructions](https://docs.anaconda.com/anaconda/install/). -After installing Anaconda, to set up a Python 3.7 environment (in case you don't have one +After installing Anaconda, to set up a Python 3.8 environment (in case you don't have one yet) execute: ```bash -conda create --yes -n newsmtsc python=3.7 +conda create --yes -n newsmtsc python=3.8 conda activate newsmtsc ``` diff --git a/setup.cfg b/setup.cfg index 9e591e8..88acfcd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = NewsSentiment -version = 1.1.25 +version = 1.2.28 author = Felix Hamborg author_email = felix.hamborg@uni-konstanz.de description = Easy-to-use, high-quality target-dependent sentiment classification for English news articles @@ -15,8 +15,10 @@ classifiers = License :: OSI Approved :: MIT License Operating System :: OS Independent Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Intended Audience :: Developers Intended Audience :: Science/Research Topic :: Scientific/Engineering @@ -28,7 +30,7 @@ classifiers = package_dir = = . packages = find_namespace: -python_requires = >=3.7,<3.9 +python_requires = >=3.8, <3.12 # include_package_data = true install_requires = boto3>=1.19.7 @@ -46,8 +48,8 @@ install_requires = spacy>=3.2 tabulate>=0.8.9 tqdm>=4.62.3 - transformers>=4.17.0,<=4.24.0 - torch>=1.12.0,<1.14.0 + transformers>=4.17,<=4.24 + torch>=1.12,<2.1 [options.packages.find] where = .