From a8e077afabc9ad1b9bd9ae2af4698b38d4d4b4f3 Mon Sep 17 00:00:00 2001 From: Simon Epstein Date: Mon, 22 Aug 2022 17:03:57 +0100 Subject: [PATCH 01/32] Fix UnboundLocalError thrown during training. NewsSentiment/train.py --own_model_name grutsc --dataset_name newsmtsc-rw threw this error as local referenced before assignment in some situations. Looks like an oversight when this variable was introduced in 213c1da0 --- NewsSentiment/train.py | 1 + 1 file changed, 1 insertion(+) diff --git a/NewsSentiment/train.py b/NewsSentiment/train.py index fb2ffb2..b8cd4a7 100644 --- a/NewsSentiment/train.py +++ b/NewsSentiment/train.py @@ -692,6 +692,7 @@ def _evaluate(self, data_loader, get_examples=False, basepath=None): t_outputs_all = None t_text_bert_indices_targets_mask_all = None t_texts_all = [] + t_outputs_confidence = None # switch model to evaluation mode self.own_model.eval() From f8869bb570f4d6e809beafd97389d91e525aca77 Mon Sep 17 00:00:00 2001 From: Felix Hamborg Date: Tue, 23 Aug 2022 10:10:53 +0200 Subject: [PATCH 02/32] fix issue mentioned in PR #19 --- NewsSentiment/train.py | 1 - setup.cfg | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/NewsSentiment/train.py b/NewsSentiment/train.py index b8cd4a7..db5629e 100644 --- a/NewsSentiment/train.py +++ b/NewsSentiment/train.py @@ -754,7 +754,6 @@ def _evaluate(self, data_loader, get_examples=False, basepath=None): y_pred = self._get_classes_from_sequence_output( t_outputs_all, t_text_bert_indices_targets_mask_all ).cpu() - t_outputs_confidence = None else: # softmax: get predictions from outputs # have to take the 3rd (dim=2) dimension diff --git a/setup.cfg b/setup.cfg index 5214a9a..c4585a0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,9 @@ [metadata] name = NewsSentiment -version = 1.1.21 +version = 1.1.23 author = Felix Hamborg author_email = felix.hamborg@uni-konstanz.de -description = Easy-to-use, high-quality target-dependent sentiment classification for news articles +description = Easy-to-use, high-quality target-dependent sentiment classification for English news articles long_description = file: READMEpypi.md long_description_content_type = text/markdown url = https://github.com/fhamborg/NewsMTSC From 89e670dda2d5ef1ae07d439fa060087ca4fec254 Mon Sep 17 00:00:00 2001 From: Victor Ananyev Date: Wed, 25 Jan 2023 18:39:34 +0100 Subject: [PATCH 03/32] dont load the spacy sm model for inference --- NewsSentiment/dataset.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/NewsSentiment/dataset.py b/NewsSentiment/dataset.py index 46089da..f654af2 100644 --- a/NewsSentiment/dataset.py +++ b/NewsSentiment/dataset.py @@ -42,15 +42,6 @@ from NewsSentiment.models.FXBaseModel import FXBaseModel logger = get_logger() -try: - nlp = spacy.load("en_core_web_sm") -except OSError: - spacy.cli.download("en_core_web_sm") - nlp = spacy.load("en_core_web_sm") - -# get list of parser's labels -parser_index = nlp.pipe_names.index("parser") -nlp_dep_parser_labels = list(nlp.pipeline[parser_index][1].labels) class RandomOversampler(torch.utils.data.sampler.Sampler): @@ -86,6 +77,7 @@ def __iter__(self): class FXEasyTokenizer: + NLP_DEP_PARSER_LABELS = None NUM_CATEGORIES_OF_SELECTED_KNOWLEDGE_SOURCES = 0 __PROCESSED_KNOWLEDGE_SOURCES = set() @@ -96,11 +88,26 @@ def __init__( knowledge_sources: Iterable[str], is_use_natural_target_phrase_for_spc: bool, ): + self._get_labels() self.tokenizers_name_and_obj = tokenizers_name_and_obj self.max_seq_len = max_seq_len self.knowledge_sources = knowledge_sources self.is_use_natural_target_phrase_for_spc = is_use_natural_target_phrase_for_spc + @classmethod + def _get_labels(cls): + if cls.NLP_DEP_PARSER_LABELS is None: + return + try: + nlp = spacy.load("en_core_web_sm") + except OSError: + spacy.cli.download("en_core_web_sm") + nlp = spacy.load("en_core_web_sm") + + # get list of parser's labels + parser_index = nlp.pipe_names.index("parser") + cls.NLP_DEP_PARSER_LABELS = list(nlp.pipeline[parser_index][1].labels) + @staticmethod def create_entire_text( text_left: str, From 99ab49099b088ddf47bcac6ddaf5324285862ac5 Mon Sep 17 00:00:00 2001 From: Victor Ananyev Date: Wed, 25 Jan 2023 18:47:49 +0100 Subject: [PATCH 04/32] fix mention --- NewsSentiment/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NewsSentiment/dataset.py b/NewsSentiment/dataset.py index f654af2..9a895da 100644 --- a/NewsSentiment/dataset.py +++ b/NewsSentiment/dataset.py @@ -267,7 +267,7 @@ def _calculate_dep_matrix(self, text_tokens, text_tokens_as_str): # offset the relation by 1 so that the root relation (which is 0) is # non-zero index_of_relation_to_head = ( - nlp_dep_parser_labels.index(relation_to_head) + 1 + self.NLP_DEP_PARSER_LABELS.index(relation_to_head) + 1 ) # insert to dependency tensor dependency_tensor[ From 49595306d7524a897c264d0ecd4e800c36b058e1 Mon Sep 17 00:00:00 2001 From: Victor Ananyev Date: Thu, 2 Feb 2023 15:46:11 +0100 Subject: [PATCH 05/32] fix missing nlp --- NewsSentiment/dataset.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/NewsSentiment/dataset.py b/NewsSentiment/dataset.py index 9a895da..250adc2 100644 --- a/NewsSentiment/dataset.py +++ b/NewsSentiment/dataset.py @@ -77,6 +77,7 @@ def __iter__(self): class FXEasyTokenizer: + NLP = None NLP_DEP_PARSER_LABELS = None NUM_CATEGORIES_OF_SELECTED_KNOWLEDGE_SOURCES = 0 __PROCESSED_KNOWLEDGE_SOURCES = set() @@ -96,17 +97,17 @@ def __init__( @classmethod def _get_labels(cls): - if cls.NLP_DEP_PARSER_LABELS is None: + if cls.NLP_DEP_PARSER_LABELS is not None: return try: - nlp = spacy.load("en_core_web_sm") + cls.NLP = spacy.load("en_core_web_sm") except OSError: spacy.cli.download("en_core_web_sm") - nlp = spacy.load("en_core_web_sm") + cls.NLP = spacy.load("en_core_web_sm") # get list of parser's labels - parser_index = nlp.pipe_names.index("parser") - cls.NLP_DEP_PARSER_LABELS = list(nlp.pipeline[parser_index][1].labels) + parser_index = cls.NLP.pipe_names.index("parser") + cls.NLP_DEP_PARSER_LABELS = list(cls.NLP.pipeline[parser_index][1].labels) @staticmethod def create_entire_text( @@ -283,7 +284,7 @@ def _calculate_dep_distance(self, text_tokens, text_left_len, target): # whitespace split as in https://github.com/StevePhan101/LCFS-BERT/ # we ensure that the same tokenization as was used for the text is applied for # the target - nlp_target = nlp(target) + nlp_target = self.NLP(target) # target_terms_lowercased = [a.lower() for a in target.split()] target_terms_lowercased = [a.text.lower() for a in nlp_target] @@ -542,7 +543,7 @@ def _create_mapping_from_tokenbased_to_wordpiece_based(self, text, tok_obj): # for spacy, we need to remove leading spaces as they will yield a single # token text_without_leading_space = text.strip() - nlp_text = nlp(text_without_leading_space) + nlp_text = self.NLP(text_without_leading_space) # get only non-single-space tokens, see # https://github.com/explosion/spaCy/issues/1707 From 8fb8dbbeb21fd6dc1e4126ae5a31f49ca492616b Mon Sep 17 00:00:00 2001 From: Felix Hamborg Date: Wed, 15 Feb 2023 08:48:18 +0100 Subject: [PATCH 06/32] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 3c44e6a..1eaffb9 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,9 @@ pool to run experiments of these argument combinations in parallel. After comple which contains detailed results, including evaluation performance, of all experiments. By using `createoverview.py`, you can export this summary into an Excel spreadsheet. +# Support +If you have questions on how to use NewsMTSC or its library, please create a new [issue](https://github.com/fhamborg/NewsMTSC/issues) on GitHub. Please understand that we are not able to provide individual support via email. We think that help is more valuable if it is shared publicly so that more people can benefit from it. + # Acknowledgements This repository is in part based on [ABSA-PyTorch](https://github.com/songyouwei/ABSA-PyTorch). We thank Song et al. for making their excellent repository open source. From cbc92e61dd11a244be1af2b12c45be79d5904461 Mon Sep 17 00:00:00 2001 From: Felix Hamborg Date: Fri, 17 Mar 2023 16:00:54 +0100 Subject: [PATCH 07/32] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1eaffb9..6aff4af 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ TSC classification performance on NewsMTSC. Check it out - it **works out of the * **I need the dataset**: you can [download it here](https://github.com/fhamborg/NewsMTSC/raw/main/NewsSentiment/controller_data/datasets/NewsMTSC-dataset/NewsMTSC-dataset.zip) or [view it here](https://github.com/fhamborg/NewsMTSC/tree/main/NewsSentiment/controller_data/datasets/NewsMTSC-dataset). We also offer NewsMTSC as a dataset on [Huggingface Hub](https://huggingface.co/datasets/fhamborg/news_sentiment_newsmtsc) and on [Kaggle](https://www.kaggle.com/fhamborg/news-articles-sentiment). * **I want to train my own models**: read the remainder of this file. +Reminder: the following description is only relevant if you in fact want to train your own models. If that's not the case, please check above for links to the dataset and our easy-to-use python package. + # Installation It's super easy, we promise! Note that following these instructions is only necessary if you're planning to train a model using our tool. If you only want to predict the sentiment of sentences, please use our [Python package](https://pypi.org/project/NewsSentiment/), which is even easier to install and use :-) From 5f8889ea9f86e1131e9cff31119bee54f4eb2879 Mon Sep 17 00:00:00 2001 From: "Tilman Hornung (t1h0)" <64684735+t1h0@users.noreply.github.com> Date: Tue, 18 Apr 2023 18:09:34 +0200 Subject: [PATCH 08/32] Update dependencies to work with torch 1.13 --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index c4585a0..487a771 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,8 +46,8 @@ install_requires = spacy>=3.2 tabulate>=0.8.9 tqdm>=4.62.3 - transformers==4.17.0 - torch==1.11.0 + transformers>=4.17.0,<=4.24.0 + torch>=1.12.0,<1.14.0 [options.packages.find] where = . From e6caed831fd3e5eccf53439124d9b4950fc34fbe Mon Sep 17 00:00:00 2001 From: Felix Hamborg Date: Tue, 9 May 2023 10:52:27 +0200 Subject: [PATCH 09/32] increase version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 487a771..f3e4ac1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = NewsSentiment -version = 1.1.23 +version = 1.1.24 author = Felix Hamborg author_email = felix.hamborg@uni-konstanz.de description = Easy-to-use, high-quality target-dependent sentiment classification for English news articles From 3a706ce322416737844f4e9637f29df5ca45aaa0 Mon Sep 17 00:00:00 2001 From: Felix Hamborg Date: Tue, 9 May 2023 10:54:19 +0200 Subject: [PATCH 10/32] increase version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index f3e4ac1..9e591e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = NewsSentiment -version = 1.1.24 +version = 1.1.25 author = Felix Hamborg author_email = felix.hamborg@uni-konstanz.de description = Easy-to-use, high-quality target-dependent sentiment classification for English news articles From 0841bc7bba35676db1b94b3a464a86d9e4b61b40 Mon Sep 17 00:00:00 2001 From: Felix Hamborg Date: Thu, 5 Oct 2023 16:09:01 +0200 Subject: [PATCH 11/32] Update READMEpypi.md --- READMEpypi.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/READMEpypi.md b/READMEpypi.md index befcc33..5c34b00 100644 --- a/READMEpypi.md +++ b/READMEpypi.md @@ -44,6 +44,11 @@ sentiment = tsc.infer_from_text("" ,"Mark Meadows", "'s coverup of Trump’s cou print(sentiment[0]) ``` +# How to identify a person in a sentence? + +In case your data is not separated as shown in the examples above, i.e., in three segments, you will need to identify one (or more) targets first. +How this is done best depends on your project and analysis task but you may, for example, use NER. This [example](https://github.com/fhamborg/NewsMTSC/issues/30#issuecomment-1700645679) shows a simple way of doing so. + # How to cite If you use the dataset or model, please cite our [paper](https://www.aclweb.org/anthology/2021.eacl-main.142/) ([PDF](https://www.aclweb.org/anthology/2021.eacl-main.142.pdf)): From 6ac63a98d99e5ba063512918d2057895950a2be8 Mon Sep 17 00:00:00 2001 From: Tilman Hornung <64684735+t1h0@users.noreply.github.com> Date: Tue, 24 Oct 2023 12:02:01 +0200 Subject: [PATCH 12/32] Bugfix index based infer --- NewsSentiment/infer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NewsSentiment/infer.py b/NewsSentiment/infer.py index 10f4316..a9c5f92 100644 --- a/NewsSentiment/infer.py +++ b/NewsSentiment/infer.py @@ -115,7 +115,7 @@ def infer( if text: text_left = text[:target_mention_from] target_mention = text[target_mention_from:target_mention_to] - text_right = text[target_mention_from:] + text_right = text[target_mention_to:] # assert text_left.endswith(' ') # we cannot handle commas, if we have this # check From 842f92b678899f0d63bb31154ae1bd121c64ecfe Mon Sep 17 00:00:00 2001 From: "Tilman Hornung (t1h0)" <64684735+t1h0@users.noreply.github.com> Date: Wed, 6 Dec 2023 15:12:17 +0100 Subject: [PATCH 13/32] Raise python to 3.8 --- README.md | 8 ++++---- pythoninfo.md | 10 +++++----- setup.cfg | 3 +-- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 6aff4af..897b2f9 100644 --- a/README.md +++ b/README.md @@ -25,18 +25,18 @@ NewsMTSC was tested on MacOS and Ubuntu; other OS may work, too. Let us know :-) **1. Setup the environment:** -This step is optional if you have Python 3.7 installed already (`python --version`). If you don't have Python 3.7, we recommend using Anaconda for setting up requirements. If you do not have it yet, follow Anaconda's +This step is optional if you have Python 3.8 installed already (`python --version`). If you don't have Python 3.8, we recommend using Anaconda for setting up requirements. If you do not have it yet, follow Anaconda's [installation instructions](https://docs.anaconda.com/anaconda/install/). -To setup a Python 3.7 environment (in case you don't have one yet) you may use, for example: +To setup a Python 3.8 environment (in case you don't have one yet) you may use, for example: ```bash -conda create --yes -n newsmtsc python=3.7 +conda create --yes -n newsmtsc python=3.8 conda activate newsmtsc ``` FYI, for users of virtualenv, the equivalent command would be: ```bash -virtualenv -ppython3.7 --setuptools 45 venv +virtualenv -ppython3.8 --setuptools 45 venv source venv/bin/activate ``` diff --git a/pythoninfo.md b/pythoninfo.md index 757afe5..deaafc4 100644 --- a/pythoninfo.md +++ b/pythoninfo.md @@ -1,13 +1,13 @@ -This step is optional if you have Python 3.7 or 3.8 installed (run `python --version` -in a terminal and check the version that is printed). If you don't have Python 3.7, we +This step is optional if you have Python 3.8 installed (run `python --version` +in a terminal and check the version that is printed). If you don't have Python 3.8, we recommend using Anaconda for setting up requirements because it is very easy (but any way -of installing Python 3.7 is fine). If you do not have Anaconda yet, follow their +of installing Python 3.8 is fine). If you do not have Anaconda yet, follow their [installation instructions](https://docs.anaconda.com/anaconda/install/). -After installing Anaconda, to set up a Python 3.7 environment (in case you don't have one +After installing Anaconda, to set up a Python 3.8 environment (in case you don't have one yet) execute: ```bash -conda create --yes -n newsmtsc python=3.7 +conda create --yes -n newsmtsc python=3.8 conda activate newsmtsc ``` diff --git a/setup.cfg b/setup.cfg index 9e591e8..b71d702 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,6 @@ classifiers = License :: OSI Approved :: MIT License Operating System :: OS Independent Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Intended Audience :: Developers Intended Audience :: Science/Research @@ -28,7 +27,7 @@ classifiers = package_dir = = . packages = find_namespace: -python_requires = >=3.7,<3.9 +python_requires = ==3.8.* # include_package_data = true install_requires = boto3>=1.19.7 From e976bb2f227429be8261bcf97a4407c073f53db8 Mon Sep 17 00:00:00 2001 From: "Tilman Hornung (t1h0)" <64684735+t1h0@users.noreply.github.com> Date: Wed, 6 Dec 2023 15:25:42 +0100 Subject: [PATCH 14/32] Batch inference Tokenizer not yet with batching --- NewsSentiment/dataset.py | 552 +++++++++++++++++++++++++-------------- NewsSentiment/infer.py | 252 ++++++++++++++---- 2 files changed, 554 insertions(+), 250 deletions(-) diff --git a/NewsSentiment/dataset.py b/NewsSentiment/dataset.py index 250adc2..ec43a99 100644 --- a/NewsSentiment/dataset.py +++ b/NewsSentiment/dataset.py @@ -1,6 +1,18 @@ import random -from collections import Counter -from typing import List, Iterable, Set +from collections import Counter, defaultdict +from typing import ( + List, + Iterable, + Set, + Union, + Dict, + overload, + Sequence, + Mapping, + Literal, + Tuple, + Optional, +) import jsonlines import networkx as nx @@ -16,6 +28,7 @@ XLNetTokenizer, AlbertTokenizer, ) +from math import ceil from NewsSentiment.SentimentClasses import SentimentClasses from NewsSentiment.customexceptions import TooLongTextException, TargetNotFoundException @@ -109,13 +122,33 @@ def _get_labels(cls): parser_index = cls.NLP.pipe_names.index("parser") cls.NLP_DEP_PARSER_LABELS = list(cls.NLP.pipeline[parser_index][1].labels) + @staticmethod + @overload + def create_entire_text( + text_left: str, + target_phrase: str, + text_right: str, + is_return_modified_text_left: Literal[True], + ) -> Tuple[str, str]: + ... + + @staticmethod + @overload + def create_entire_text( + text_left: str, + target_phrase: str, + text_right: str, + is_return_modified_text_left: Literal[False], + ) -> str: + ... + @staticmethod def create_entire_text( text_left: str, target_phrase: str, text_right: str, is_return_modified_text_left: bool, - ): + ) -> Union[Tuple[str, str], str]: """ Create the entire text from the three prepared text segment. Will modify text_left (append a whitespace " ") if it is empty, which is necessary for @@ -152,8 +185,7 @@ def prepare_left_segment(text_left: str): :param text_left: :return: """ - while len(text_left) > 0 and text_left[0] == "\n": - text_left = text_left[1:] + text_left = text_left.lstrip("\n") if len(text_left) > 0 and text_left[-1] != " ": text_left += " " return text_left @@ -165,9 +197,7 @@ def prepare_target_mention(target_mention: str): :param target_mention: :return: """ - while len(target_mention) > 0 and target_mention[0] == "\n": - target_mention = target_mention[1:] - return target_mention + return target_mention.lstrip("\n") @staticmethod def prepare_right_segment(text_right: str): @@ -516,8 +546,8 @@ def _create_knowledge_source_tensor( self.max_seq_len, num_categories, dtype=torch.long ) for word_index, word in enumerate(text_tokens_as_str, start=offset): - positions_of_current_word_in_input_sequence = self._map_token_index_to_wordpiece_index( - mapping, word_index + positions_of_current_word_in_input_sequence = ( + self._map_token_index_to_wordpiece_index(mapping, word_index) ) if mode == "nrc_emotions": emotion_tensor = get_nrc_emotions_as_tensor(word) @@ -600,7 +630,10 @@ def _create_dependency_tensor( dependency_column = dependency_tensor_of_tokens[word_index].tolist() dependency_column_ones = self._convert_non_null_to_one(dependency_column) _check_num_heads = sum(dependency_column_ones) - assert _check_num_heads in [0, 1,], ( + assert _check_num_heads in [ + 0, + 1, + ], ( f"expected only zero or one heads, found {_check_num_heads} for word " f"{word} in {text_tokens_as_str}" ) @@ -617,11 +650,15 @@ def _create_dependency_tensor( dependency_type = dependency_column[head_index] assert dependency_type != 0 - positions_of_current_word_in_input_sequence = self._map_token_index_to_wordpiece_index( - mapping_token2wordpiece, word_index_withoffset + positions_of_current_word_in_input_sequence = ( + self._map_token_index_to_wordpiece_index( + mapping_token2wordpiece, word_index_withoffset + ) ) - positions_of_current_head_in_input_sequence = self._map_token_index_to_wordpiece_index( - mapping_token2wordpiece, head_index_withoffset + positions_of_current_head_in_input_sequence = ( + self._map_token_index_to_wordpiece_index( + mapping_token2wordpiece, head_index_withoffset + ) ) for position_word in positions_of_current_word_in_input_sequence: @@ -669,8 +706,10 @@ def _create_dependency_tree_hop_distances_of_tokens_to_target( ): word_index_without_offset = word_index - mapping_token2wordpiece_offset - positions_of_current_word_in_input_sequence = self._map_token_index_to_wordpiece_index( - mapping_token2wordpiece, word_index + positions_of_current_word_in_input_sequence = ( + self._map_token_index_to_wordpiece_index( + mapping_token2wordpiece, word_index + ) ) depdistance_of_current_word_in_input_sequence = dist[ @@ -686,194 +725,316 @@ def _create_dependency_tree_hop_distances_of_tokens_to_target( def create_model_input_seqs( self, - text_left, - target_phrase, - text_right, - coreferential_targets_for_target_mask: Iterable[dict], - ): + text_left: str, + target_phrase: str, + text_right: str, + coreferential_targets_for_target_mask: Optional[Iterable[dict]], + ) -> Mapping[str, Mapping[str, Union[torch.Tensor, bool]]]: """ Creates input sequences for a given target. Prior components have processed the jsonl so that coreferential_targets_for_target_mask will contain a list of coreferential mentions of the target if and only if coref_mode == "in_targetmask". In that case, will produce an individual target mask for each coref mention and merge them to the target mask of the preferred mention. - :param text_left: - :param target_phrase: - :param text_right: - :param further_mentions: - :return: """ - text = self.create_entire_text( - text_left, target_phrase, text_right, is_return_modified_text_left=False + assert all( + isinstance(arg, str) for arg in (text_left, target_phrase, text_right) + ), "Wrong input types." + + return self.create_model_input_seqs_batch( + (text_left, target_phrase, text_right), + coreferential_targets_for_target_mask=( + coreferential_targets_for_target_mask, + ), + single_target_output=True, ) - logger.debug(f"'{text_left}' '{target_phrase}' '{text_right}'") - input_seqs_per_tokenizer = {} + @overload + def create_model_input_seqs_batch( + self, + *targets: ..., + coreferential_targets_for_target_mask: ..., + batch_size: int = ..., + single_target_output: Literal[False] = ..., + ) -> List[Mapping[str, Mapping[str, Union[torch.Tensor, Tuple[bool], bool]]]]: + ... + + @overload + def create_model_input_seqs_batch( + self, + *targets: ..., + coreferential_targets_for_target_mask: ..., + batch_size: int = ..., + single_target_output: Literal[True] = ..., + ) -> Mapping[str, Mapping[str, Union[torch.Tensor, bool]]]: + ... + + def create_model_input_seqs_batch( + self, + *targets: Tuple[str, str, str], + coreferential_targets_for_target_mask: Optional[ + Sequence[Optional[Iterable[dict]]] + ], + batch_size: int = 1, + single_target_output: bool = False, + ) -> Union[ + List[Mapping[str, Mapping[str, Union[torch.Tensor, Tuple[bool], bool]]]], + Mapping[str, Mapping[str, Union[torch.Tensor, bool]]], + ]: + """ + Creates input sequences for given targets. Prior components have processed the jsonl + so that coreferential_targets_for_target_mask will contain a list of coreferential + mentions of the targets if and only if coref_mode == "in_targetmask". In that case, + will produce an individual target mask for each coref mention and merge them to + the target mask of the preferred mention. - for name, tok_obj in self.tokenizers_name_and_obj.items(): - logger.debug(f"{name}") + NEW + Multiple target inputs and batching. Output will be a tuple of batches with the + created sequences being of shape [batch_size,seq_length]. To output a single + target without batch output and with tensors of shape [seq_length], + set squeeze_single_target = True. + """ - # text - text_ids_with_special_tokens = tok_obj.encode( - text, - max_length=self.max_seq_len, - pad_to_max_length=True, - add_special_tokens=True, - truncation="longest_first", - ) - # text SEP target - adjusted_target_phrase = target_phrase - if self.is_use_natural_target_phrase_for_spc: - adjusted_target_phrase = "What do you think of " + target_phrase + "?" - text_then_target_ids_with_special_tokens_dict = tok_obj.encode_plus( - text, - text_pair=adjusted_target_phrase, - max_length=self.max_seq_len, - pad_to_max_length=True, - add_special_tokens=True, - truncation="longest_first", - ) - text_then_target_ids_with_special_tokens_dict = ( - text_then_target_ids_with_special_tokens_dict.data - ) - text_then_target_ids_with_special_tokens = text_then_target_ids_with_special_tokens_dict[ - "input_ids" - ] - if type(tok_obj) == RobertaTokenizer: - # roberta doesnt have segments, so we produce fake segment ids here, and later simply dont pass them - text_then_target_ids_with_special_tokens_segment_ids = [0] * len( - text_then_target_ids_with_special_tokens - ) - else: - text_then_target_ids_with_special_tokens_segment_ids = text_then_target_ids_with_special_tokens_dict[ - "token_type_ids" - ] - # target - target_ids_with_special_tokens = tok_obj.encode( - target_phrase, - max_length=self.max_seq_len, - pad_to_max_length=True, - add_special_tokens=True, - truncation="longest_first", - ) + num_targets = len(targets) + num_batches = ceil(num_targets / batch_size) - # create target masks - target_mask_seq_for_text_with_special_tokens = self._create_target_mask( - tok_obj, - text_left, - target_phrase, - text_right, - for_text_with_special_tokens=True, - ) - # create target mask for coreferential targets of given target - coref_target_masks = self._create_coreferential_target_masks( - tok_obj, coreferential_targets_for_target_mask + if not coreferential_targets_for_target_mask: + coreferential_targets_for_target_mask = [None] * num_targets + elif num_targets != len(coreferential_targets_for_target_mask): + raise TypeError( + "Number of coreferential_targets_for_target_mask must match number of targets." ) - # merge them into the preferred target mask - merged_target_mask = self._merge_coref_target_masks_into_preferred_target_mask( - target_mask_seq_for_text_with_special_tokens, coref_target_masks - ) - target_mask_seq_for_text_with_special_tokens = merged_target_mask - # create also text ids without max length to see if the one that will be - # used was truncated - text_ids_with_special_tokens_no_max_length = tok_obj.encode( - text, add_special_tokens=True - ) - text_num_truncated_tokens = len( - text_ids_with_special_tokens_no_max_length - ) - len(text_ids_with_special_tokens) - - # iterate tokens of full text and look up in dicts - # create mapping from token-based indexes to wordpiece-based indexes - # the function will also remove any tokens for which the word piece index - # would be after max seq len - # under certain circumstances, e.g., when using albert as well as the target - # phrase is close to what would be cut off (but is not at this point), it can happen - # that at a later point in time, i.e., _create_dependency_tree_hop_distances_of_tokens_to_target - # or more specifically _calculate_dep_distance, the target cannot be found in the list of - # tokens anymore, because this list of now cut off. thus, i changed the assert in - # _calculate_dep_distance to throwing an exception that is catched later on - ( - mapping_token2wordpiece, - mapping_token2wordpiece_offset, - text_tokens_as_str, - text_tokens, - ) = self._create_mapping_from_tokenbased_to_wordpiece_based(text, tok_obj) - - # create additional knowledge source tensors - # stack only those knowledge sources that were requested by arguments - selected_tensors_knowledge_sources_text = [] - for source in self.knowledge_sources: - text_tensor = self._create_knowledge_source_tensor( - text_tokens_as_str, - mapping_token2wordpiece_offset, - mapping_token2wordpiece, - source, - ) - selected_tensors_knowledge_sources_text.append(text_tensor) - if self.knowledge_sources: - text_stacked_knowledge_source_info = torch.cat( - tuple(selected_tensors_knowledge_sources_text), dim=1 - ) + single_output = False + if single_target_output: + if num_targets == 1: + single_output = True else: - text_stacked_knowledge_source_info = None - - # syntax hop distance - text_dependency_tree_hop_distances = self._create_dependency_tree_hop_distances_of_tokens_to_target( - text_tokens, - len(text_left), - target_phrase, - text_tokens_as_str, - mapping_token2wordpiece, - mapping_token2wordpiece_offset, - ) + logger.warning( + """Multiple targets: single_target_output (is True) will be ignored! + Consider using create_model_input_seqs() instead.""" + ) + + out: List[Mapping[str, Mapping[str, Union[torch.Tensor, bool]]]] = [] - text_dependency_matrix = self._create_dependency_tensor( - text_tokens, - text_tokens_as_str, - mapping_token2wordpiece, - mapping_token2wordpiece_offset, + for batch_number in range(num_batches): + batch_result = {} + batch_start = batch_number * batch_size + batch_end = ( + None + if batch_number == num_batches - 1 + else (batch_number + 1) * batch_size ) - # create item with indexes and masks - input_seqs_per_tokenizer[name] = { - FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( - text_ids_with_special_tokens - ), - FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS_TARGET_MASK: torch.FloatTensor( - target_mask_seq_for_text_with_special_tokens - ), - FIELD_IS_OVERFLOW: text_num_truncated_tokens > 0, - FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( - text_then_target_ids_with_special_tokens - ), - # we used to have text-then-target target mask here, but won't use it, - # since it would be identical to the text target mask (since we only - # want to mark the target within the text, but not in the 2nd target - # component) - # FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS_TARGET_MASK: torch.LongTensor( - # text_then_target_ids_with_special_tokens_target_mask, - # ), - FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS_SEGMENT_IDS: torch.LongTensor( - text_then_target_ids_with_special_tokens_segment_ids - ), - FIELD_TARGET_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( - target_ids_with_special_tokens - ), - FIELD_SYNTAX_HOP_DISTANCE_TO_TARGET: text_dependency_tree_hop_distances, - FIELD_SYNTAX_DEPENDENCY_MATRIX: text_dependency_matrix, - } - - # add knowledge source if requested - if self.knowledge_sources: - input_seqs_per_tokenizer[name][ - FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS_SELECTED_KNOWLEDGE_SOURCES - ] = text_stacked_knowledge_source_info - # likewise as for FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS_TARGET_MASK, we - # don't need to set a special knowledge source mask for text-then-target - - return input_seqs_per_tokenizer + for tok_name, tok_obj in self.tokenizers_name_and_obj.items(): + logger.debug(f"{tok_name}") + + tokenizer_result = defaultdict(list) + + for (left, target_phrase, right), corefs in zip( + targets[batch_start:batch_end], + coreferential_targets_for_target_mask[batch_start:batch_end], + ): + text = self.create_entire_text( + left, target_phrase, right, is_return_modified_text_left=False + ) + logger.debug(f"'{left}' '{target_phrase}' '{right}'") + + # text + text_ids_with_special_tokens = tok_obj.encode( + text, + max_length=self.max_seq_len, + pad_to_max_length=True, + add_special_tokens=True, + truncation="longest_first", + ) + # text SEP target + adjusted_target_phrase = target_phrase + if self.is_use_natural_target_phrase_for_spc: + adjusted_target_phrase = ( + "What do you think of " + target_phrase + "?" + ) + text_then_target_ids_with_special_tokens_dict = tok_obj.encode_plus( + text, + text_pair=adjusted_target_phrase, + max_length=self.max_seq_len, + pad_to_max_length=True, + add_special_tokens=True, + truncation="longest_first", + ) + text_then_target_ids_with_special_tokens_dict = ( + text_then_target_ids_with_special_tokens_dict.data + ) + text_then_target_ids_with_special_tokens = ( + text_then_target_ids_with_special_tokens_dict["input_ids"] + ) + if type(tok_obj) == RobertaTokenizer: + # roberta doesnt have segments, so we produce fake segment ids + # here, and later simply dont pass them + text_then_target_ids_with_special_tokens_segment_ids = [ + 0 + ] * len(text_then_target_ids_with_special_tokens) + else: + text_then_target_ids_with_special_tokens_segment_ids = ( + text_then_target_ids_with_special_tokens_dict[ + "token_type_ids" + ] + ) + # target + target_ids_with_special_tokens = tok_obj.encode( + target_phrase, + max_length=self.max_seq_len, + pad_to_max_length=True, + add_special_tokens=True, + truncation="longest_first", + ) + + # create target masks + target_mask_seq_for_text_with_special_tokens = ( + self._create_target_mask( + tok_obj, + left, + target_phrase, + right, + for_text_with_special_tokens=True, + ) + ) + if corefs: + # create target mask for coreferential targets of given target + coref_target_masks = self._create_coreferential_target_masks( + tok_obj, corefs + ) + # merge them into the preferred target mask + merged_target_mask = ( + self._merge_coref_target_masks_into_preferred_target_mask( + target_mask_seq_for_text_with_special_tokens, + coref_target_masks, + ) + ) + target_mask_seq_for_text_with_special_tokens = ( + merged_target_mask + ) + + # create also text ids without max length to see if the one that will be + # used was truncated + text_ids_with_special_tokens_no_max_length = tok_obj.encode( + text, add_special_tokens=True + ) + text_num_truncated_tokens = len( + text_ids_with_special_tokens_no_max_length + ) - len(text_ids_with_special_tokens) + + # iterate tokens of full text and look up in dicts + # create mapping from token-based indexes to wordpiece-based indexes + # the function will also remove any tokens for which the word piece index + # would be after max seq len + # under certain circumstances, e.g., when using albert as well as the target + # phrase is close to what would be cut off (but is not at this point), it can happen + # that at a later point in time, i.e., _create_dependency_tree_hop_distances_of_tokens_to_target + # or more specifically _calculate_dep_distance, the target cannot be found in the list of + # tokens anymore, because this list of now cut off. thus, i changed the assert in + # _calculate_dep_distance to throwing an exception that is catched later on + ( + mapping_token2wordpiece, + mapping_token2wordpiece_offset, + text_tokens_as_str, + text_tokens, + ) = self._create_mapping_from_tokenbased_to_wordpiece_based( + text, tok_obj + ) + + # create additional knowledge source tensors + # stack only those knowledge sources that were requested by arguments + selected_tensors_knowledge_sources_text = [] + for source in self.knowledge_sources: + text_tensor = self._create_knowledge_source_tensor( + text_tokens_as_str, + mapping_token2wordpiece_offset, + mapping_token2wordpiece, + source, + ) + selected_tensors_knowledge_sources_text.append(text_tensor) + if self.knowledge_sources: + text_stacked_knowledge_source_info = torch.cat( + tuple(selected_tensors_knowledge_sources_text), dim=1 + ) + else: + text_stacked_knowledge_source_info = None + + # syntax hop distance + text_dependency_tree_hop_distances = ( + self._create_dependency_tree_hop_distances_of_tokens_to_target( + text_tokens, + len(left), + target_phrase, + text_tokens_as_str, + mapping_token2wordpiece, + mapping_token2wordpiece_offset, + ) + ) + + text_dependency_matrix = self._create_dependency_tensor( + text_tokens, + text_tokens_as_str, + mapping_token2wordpiece, + mapping_token2wordpiece_offset, + ) + + # create item with indexes and masks + result = { + FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( + text_ids_with_special_tokens + ), + FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS_TARGET_MASK: torch.FloatTensor( + target_mask_seq_for_text_with_special_tokens + ), + FIELD_IS_OVERFLOW: text_num_truncated_tokens > 0, + FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( + text_then_target_ids_with_special_tokens + ), + FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS_SEGMENT_IDS: torch.LongTensor( + text_then_target_ids_with_special_tokens_segment_ids + ), + FIELD_TARGET_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( + target_ids_with_special_tokens + ), + FIELD_SYNTAX_HOP_DISTANCE_TO_TARGET: text_dependency_tree_hop_distances, + FIELD_SYNTAX_DEPENDENCY_MATRIX: text_dependency_matrix, + } + + # add knowledge source if requested + if self.knowledge_sources: + result[ + FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS_SELECTED_KNOWLEDGE_SOURCES + ] = text_stacked_knowledge_source_info + + # add result to tokenizer result + if single_output: + tokenizer_result.update(result) + else: + for field_name, field_content in result.items(): + tokenizer_result[field_name].append(field_content) + + if not single_output: + # stack tensors in tokenizer result + for field_name, field_content in tokenizer_result.items(): + tokenizer_result[field_name] = ( + torch.stack(field_content) + if isinstance(field_content[0], torch.Tensor) + else tuple(field_content) + ) + + # add tokenizer result to batch result + batch_result[tok_name] = dict(tokenizer_result) + + if single_output: + # return the single target result if requested + return batch_result + + # add batch result to output + out.append(batch_result) + + return out def _create_coreferential_target_masks( self, tok_obj, coreferential_targets_for_target_mask @@ -1110,7 +1271,10 @@ def _create_target_inputs( target_mention = FXEasyTokenizer.prepare_target_mention(target_mention) text_right = FXEasyTokenizer.prepare_right_segment(text_right) text = FXEasyTokenizer.create_entire_text( - text_left, target_mention, text_right, is_return_modified_text_left=False, + text_left, + target_mention, + text_right, + is_return_modified_text_left=False, ) # text to indexes @@ -1393,8 +1557,10 @@ def task_to_dataset_item(self, task, coref_mode: str, ignore_parsing_errors): # if the coref mode (only during training though) is additional_examples, repeat them here if coref_mode == "additional_examples": - expanded_targets = self._expand_coref_mentions_of_targets_to_multiple_targets( - targets, text + expanded_targets = ( + self._expand_coref_mentions_of_targets_to_multiple_targets( + targets, text + ) ) elif coref_mode == "in_targetmask": expanded_targets = targets @@ -1420,8 +1586,10 @@ def task_to_dataset_item(self, task, coref_mode: str, ignore_parsing_errors): example_id, text, expanded_targets ) else: - items = self._convert_multi_targets_in_single_item_to_k_targets_in_multi_items( - example_id, text, expanded_targets + items = ( + self._convert_multi_targets_in_single_item_to_k_targets_in_multi_items( + example_id, text, expanded_targets + ) ) # iterate (virtual) items of this actual row in the jsonl and create model items diff --git a/NewsSentiment/infer.py b/NewsSentiment/infer.py index 10f4316..a1c8c67 100644 --- a/NewsSentiment/infer.py +++ b/NewsSentiment/infer.py @@ -4,6 +4,7 @@ import torch.nn.functional as F from jsonlines import jsonlines from tqdm import tqdm +from typing import overload, Any, List, Dict, Union, Sequence, Tuple, Optional, Literal from NewsSentiment.SentimentClasses import SentimentClasses from NewsSentiment.dataset import FXEasyTokenizer @@ -83,74 +84,207 @@ def infer_from_text(self, left, target, right): """ return self.infer(text_left=left, target_mention=target, text_right=right) + @overload def infer( self, - text_left: str = None, - target_mention: str = None, - text_right: str = None, - text: str = None, - target_mention_from: int = None, - target_mention_to: int = None, - ): - """ - Calculates sentiment as to target_mention in a text that is a concatenation of - text_left, - target_mention, and text_right. Note that text_left and text_right should end - with a space (or comma, etc.)), - or end with a space, respectively. Alternatively, the target can be selected via - target_mention_from and target_mention_to in text. + text_left: str = ..., + target_mention: str = ..., + text_right: str = ..., + text: None = None, + target_mention_from: None = None, + target_mention_to: None = None, + targets: None = None, + batch_size: int = ..., + ) -> Tuple[Dict[str, Any], ...]: + ... + + @overload + def infer( + self, + text_left: None = None, + target_mention: None = None, + text_right: None = None, + text: str = ..., + target_mention_from: int = ..., + target_mention_to: int = ..., + targets: None = None, + batch_size: int = ..., + ) -> Tuple[Dict[str, Any], ...]: + ... + + @overload + def infer( + self, + text_left: None = None, + target_mention: None = None, + text_right: None = None, + text: None = None, + target_mention_from: None = None, + target_mention_to: None = None, + targets: Sequence[Union[Tuple[str, str, str], Tuple[str, int, int]]] = ..., + batch_size: int = ..., + ) -> List[Tuple[Dict[str, Any], ...]]: + ... + + def infer( + self, + text_left: Optional[str] = None, + target_mention: Optional[str] = None, + text_right: Optional[str] = None, + text: Optional[str] = None, + target_mention_from: Optional[int] = None, + target_mention_to: Optional[int] = None, + targets: Optional[ + Sequence[Union[Tuple[str, str, str], Tuple[str, int, int]]] + ] = None, + batch_size: int = 1, + ) -> Union[Tuple[Dict[str, Any], ...], List[Tuple[Dict[str, Any], ...]]]: + """Computes sentiment for a target mention. + + Note that the text before and after should end with a space (or comma, etc.)), + or begin with a space, respectively. + + Args: + text_left (str | None, optional): Text before the target mention. + Defaults to None. + target_mention (str | None, optional): Target mention. Defaults to None. + text_right (str | None, optional): Text after the target mention. + Defaults to None. + text (str | None, optional): Text containing the target mention. + Defaults to None. + target_mention_from (str | None, optional): Start index of the + target mention. Defaults to None. + target_mention_to (str | None, optional): End index of the target mention. + Defaults to None. + targets (Sequence[Tuple[str, str, str] | Tuple[str, int, int]] | None, + optional): Tuples containing text_left,target_mention,text_right + or text,target_mention_from,target_mention_to for multiple targets + (mixed style is possible). Defaults to None. + batch_size (int, optional): Preferred size of batches to compute multiple + targets in. Defaults to 1. + + Returns: + Tuple[Dict[str, Any], ...] | List[Tuple[Dict[str, Any], ...]: Tuple (or + list of tuples for multiple targets with order preserved) containing + class probabilities as dictionaries with keys "class_id", "class_label" + and "class_prob". """ - is_index_based = ( - text is not None - and target_mention_from is not None - and target_mention_to is not None - ) - is_component_based = ( - text_left is not None - and target_mention is not None - and text_right is not None + component_base = (text_left, target_mention, text_right) + is_component_based = all(isinstance(i, str) for i in component_base) + + index_base = (text, target_mention_from, target_mention_to) + is_index_based = all( + isinstance(arg, typ) for arg, typ in zip(index_base, (str, int, int)) ) - assert is_index_based != is_component_based - - if text: - text_left = text[:target_mention_from] - target_mention = text[target_mention_from:target_mention_to] - text_right = text[target_mention_from:] - - # assert text_left.endswith(' ') # we cannot handle commas, if we have this - # check - assert not target_mention.startswith(" ") and not target_mention.endswith( - " " - ), f"target_mention={target_mention}; text={text}" - # assert text_right.startswith(' ') - - text_left = FXEasyTokenizer.prepare_left_segment(text_left) - target_mention = FXEasyTokenizer.prepare_target_mention(target_mention) - text_right = FXEasyTokenizer.prepare_right_segment(text_right) - - indexed_example = self.tokenizer.create_model_input_seqs( - text_left, target_mention, text_right, [] + + is_targets_based = targets is not None + + # verify input + assert ( + sum((is_component_based, is_index_based, is_targets_based)) == 1 + ), """Wrong input types or too many inputs! + Must be either one single or multiple component or index based targets.""" + + return ( + self.infer_batch( + component_base if is_component_based else index_base, batch_size=1 + )[0] + if not is_targets_based + else self.infer_batch(*targets, batch_size=batch_size) ) - inputs = self.instructor.select_inputs(indexed_example, is_single_item=True) - # invoke model - outputs = self.model(inputs) - class_probabilites = F.softmax(outputs, dim=-1).reshape((3,)).cpu().tolist() + def infer_batch( + self, + *targets: Union[Tuple[str, str, str], Tuple[str, int, int]], + batch_size: int = 1, + ) -> List[Tuple[Dict[str, Any], ...]]: + """Computes sentiment for multiple targets in batches of batch_size. + Targets are tuples of text before target, target mention, and text after target. - classification_result = [] - for class_id, class_prob in enumerate(class_probabilites): - classification_result.append( - { - "class_id": class_id, - "class_label": self.polarities_inverse[class_id], - "class_prob": class_prob, - } + Args: + *targets (Tuple[str,str,str] | Tuple[str,int,int]): Targets to compute + sentiment for. Tuples contain (text before, target mention, text after) + or (text,target mention start index, target mention end index). + Texts before and after the target mention should end with a space + (or comma, etc.)), or begin with a space, respectively. + batch_size (int, optional): Preferred size of batches to comppute the targets + in (vectorized computation). Defaults to 1. + + Returns: + List[Tuple[Dict[str,Any], ...]]: List of target classification tuples, + containing class probabilities as dictionaries with keys + "class_id", "class_label" and "class_prob". The order of tuples matches the + order of input targets. + """ + targets_prepared = [] + + for target in targets: + assert len(target) == 3, f"{target} is missing one ore more components." + + if all(isinstance(component, str) for component in target): + text_left, target_mention, text_right = target + elif all( + isinstance(component, typ) + for component, typ in zip(target, (str, int, int)) + ): + text = target[0] + target_mention_from = target[1] + target_mention_to = target[2] + text_left = text[:target_mention_from] + target_mention = text[target_mention_from:target_mention_to] + text_right = text[target_mention_to:] + else: + raise TypeError("Wrong input types.") + + # assert text_left.endswith(' ') # we cannot handle commas, if we have this + # check + assert not any( + (target_mention.startswith(" "), target_mention.endswith(" ")) + ), f"target_mention={target_mention}; text={text}" + # assert text_right.startswith(' ') + + targets_prepared.append( + ( + FXEasyTokenizer.prepare_left_segment(text_left), + FXEasyTokenizer.prepare_target_mention(target_mention), + FXEasyTokenizer.prepare_right_segment(text_right), + ) ) - classification_result = sorted( - classification_result, key=lambda x: x["class_prob"], reverse=True + indexed_examples = self.tokenizer.create_model_input_seqs_batch( + *targets_prepared, + coreferential_targets_for_target_mask=None, + batch_size=batch_size, + single_target_output=False, ) + classification_result = [] + + for batch in indexed_examples: + inputs = self.instructor.select_inputs(batch, is_single_item=False) + + outputs = self.model(inputs) + + class_probabilities_per_target = F.softmax(outputs, dim=-1).cpu().tolist() + + classification_result.extend( + tuple( + sorted( + ( + { + "class_id": class_id, + "class_label": self.polarities_inverse[class_id], + "class_prob": class_prob, + } + for class_id, class_prob in enumerate(class_probabilities) + ), + key=lambda x: x["class_prob"], + reverse=True, + ) + ) + for class_probabilities in class_probabilities_per_target + ) + return classification_result def get_info_for_label(self, classification_result, label): @@ -183,7 +317,9 @@ def parse_arguments(override_args=False): "CPU", ) parser.add_argument( - "--logging", type=str, default="ERROR", + "--logging", + type=str, + default="ERROR", ) # if own_args == None -> parse_args will use sys.argv From a93fe6335e366c383cdfb37986d09c71c6aa8a9c Mon Sep 17 00:00:00 2001 From: "Tilman Hornung (t1h0)" <64684735+t1h0@users.noreply.github.com> Date: Thu, 7 Dec 2023 14:54:31 +0100 Subject: [PATCH 15/32] Batch tokenization sub-functions don't consider batch_size yet --- NewsSentiment/dataset.py | 627 ++++++++++++++++++++++++--------------- NewsSentiment/infer.py | 10 +- 2 files changed, 388 insertions(+), 249 deletions(-) diff --git a/NewsSentiment/dataset.py b/NewsSentiment/dataset.py index ec43a99..6bdf143 100644 --- a/NewsSentiment/dataset.py +++ b/NewsSentiment/dataset.py @@ -213,53 +213,82 @@ def prepare_right_segment(text_right: str): def _create_word_to_wordpiece_mapping( self, tokenizer, words: List, for_text_with_special_tokens, offset + ): + batch_output = self._batch_create_word_to_wordpiece_mapping( + tokenizer, [words], for_text_with_special_tokens, [offset] + ) + return tuple(output[0] for output in batch_output) + + def _batch_create_word_to_wordpiece_mapping( + self, + tokenizer, + words_per_target: List[List], + for_text_with_special_tokens, + offsets, ): if not for_text_with_special_tokens: raise NotImplementedError() - previous_words = [] - mapping = None - # offset word_index by 1 (0th word has index 1) - # words_without_single_whitespace = [word for word in words if word != " "] - for word_index, word in enumerate(words, start=offset): - # update left and previous words - left = " ".join(previous_words) + " " - previous_words.append(word) - - # produce target masks - target_mask = self._create_target_mask( - tokenizer=tokenizer, - text_left=left, - target=word, - text_right="", - for_text_with_special_tokens=for_text_with_special_tokens, - is_raise_exception_if_target_after_max_seq_len=False, - ) + targets_per_target = [] + previous_words_per_target = [] - # if the current word is after the max seq len, the target mask = "toolong" - if target_mask == "toolong": - # we abort the for loop since all remaining words will also be - # after the max seq len - break + for target in words_per_target: + target_previous_words = [] + + # words_without_single_whitespace = [word for word in words if word != " "] + for word in target: + # update left and previous words + targets_per_target.append( + ( + " ".join(target_previous_words) + " ", # left + word, # target + "", # right + ) + ) + target_previous_words.append(word) + + previous_words_per_target.append(target_previous_words) + + # produce target masks + target_masks = self._batch_create_target_mask( + tokenizer=tokenizer, + targets=targets_per_target, + for_text_with_special_tokens=for_text_with_special_tokens, + is_raise_exception_if_target_after_max_seq_len=False, + ) + + target_masks = torch.LongTensor(target_masks) + # multiply by word_indices (including offset) + target_masks = torch.mul( + target_masks, + torch.tensor( + tuple( + range(offset, offset + len(wpt)) + for wpt, offset in zip(words_per_target, offsets) + ) + ).reshape((-1, 1)), + ) - target_mask = torch.LongTensor(target_mask) - target_mask = torch.mul(target_mask, word_index) - if mapping is None: - mapping = target_mask + # split back into targets + wordpiece_mappings_per_target = target_masks.split( + [len(t) for t in words_per_target], dim=0 + ) + out = [] + for mapping in wordpiece_mappings_per_target: + # make sure there is no overlap + if type(tokenizer) == RobertaTokenizer: + if mapping.prod(0).sum() > 0: + logger.debug( + "overlap when mapping tokens to wordpiece (allow overwriting because" + " Roberta is used)" + ) else: - # make sure there is no overlap - if type(tokenizer) == RobertaTokenizer: - if sum(mapping * target_mask) > 0: - logger.debug( - "overlap when mapping tokens to wordpiece (allow overwriting because " - "Roberta is used)" - ) - else: - assert sum(mapping * target_mask) == 0 - mapping = mapping + target_mask + assert mapping.prod(0).sum() == 0 + mapping = mapping.sum(0) + assert mapping.shape[0] <= self.max_seq_len + out.append(mapping) - assert mapping.shape[0] <= self.max_seq_len - return mapping, previous_words + return out, previous_words_per_target def _calculate_dep_matrix(self, text_tokens, text_tokens_as_str): _check_len_doc = len(text_tokens) @@ -410,24 +439,76 @@ def _create_target_mask( :param text_pair: if not None, will be appended to the input sequence as the second pair :return: """ + return self._batch_create_target_mask( + tokenizer, + [(text_left, target, text_right)], + for_text_with_special_tokens, + is_raise_exception_if_target_after_max_seq_len, + )[0] + + def _batch_create_target_mask( + self, + tokenizer, + targets, + for_text_with_special_tokens: bool, + is_raise_exception_if_target_after_max_seq_len: bool = True, + ): if not for_text_with_special_tokens: raise NotImplementedError() - assert target != " ", "passed a single whitespace as target" - text, text_left = self.create_entire_text( - text_left, target, text_right, is_return_modified_text_left=True + lefts = [] + rights = [] + full_texts = [] + target_phrases = [] + for target in targets: + ft, l = self.create_entire_text(*target, is_return_modified_text_left=True) + lefts.append(l) + full_texts.append(ft) + target_phrases.append(target[1]) + rights.append(target[2]) + + encodings = self._encode_for_target_mask( + tokenizer, lefts, target_phrases, rights, full_texts ) + return [ + self._create_target_mask_on_encoding( + tokenizer, *encoding, is_raise_exception_if_target_after_max_seq_len + ) + for encoding in zip(*encodings) + ] + + def _encode_for_target_mask( + self, tokenizer, text_lefts, targets, text_rights, full_texts + ): # get token ids, cf. https://huggingface.co/transformers/glossary.html - text_left_ids_with_special_tokens = tokenizer.encode( - text_left, add_special_tokens=True - ) - text_right_ids = tokenizer.encode(text_right, add_special_tokens=False) - text_ids_with_special_tokens = tokenizer.encode(text, add_special_tokens=True) - target_phrase_ids_with_special_tokens = tokenizer.encode( - target, add_special_tokens=True + text_left_ids_with_special_tokens = tokenizer( + text_lefts, add_special_tokens=True + )["input_ids"] + text_right_ids = tokenizer(text_rights, add_special_tokens=False)["input_ids"] + text_ids_with_special_tokens = tokenizer(full_texts, add_special_tokens=True)[ + "input_ids" + ] + target_phrase_ids_with_special_tokens = tokenizer( + targets, add_special_tokens=True + )["input_ids"] + + return ( + text_left_ids_with_special_tokens, + target_phrase_ids_with_special_tokens, + text_right_ids, + text_ids_with_special_tokens, ) + def _create_target_mask_on_encoding( + self, + tokenizer, + text_left_ids_with_special_tokens, + target_phrase_ids_with_special_tokens, + text_right_ids, + text_ids_with_special_tokens, + is_raise_exception_if_target_after_max_seq_len, + ): len_text_left_ids_with_special_tokens = len(text_left_ids_with_special_tokens) len_text_right_ids = len(text_right_ids) len_text_id_with_special_tokens = len(text_ids_with_special_tokens) @@ -569,28 +650,40 @@ def _create_knowledge_source_tensor( return emotions_for_sequence def _create_mapping_from_tokenbased_to_wordpiece_based(self, text, tok_obj): + batch_output = self._batch_create_mapping_from_tokenbased_to_wordpiece_based( + (text,), tok_obj + ) + return tuple(output[0] for output in batch_output) + + def _batch_create_mapping_from_tokenbased_to_wordpiece_based(self, texts, tok_obj): offset = 10 # for spacy, we need to remove leading spaces as they will yield a single # token - text_without_leading_space = text.strip() - nlp_text = self.NLP(text_without_leading_space) + text_without_leading_space = [text.strip() for text in texts] + nlp_text_per_target = self.NLP.pipe(text_without_leading_space) # get only non-single-space tokens, see # https://github.com/explosion/spaCy/issues/1707 - text_tokens = [token for token in nlp_text if not token.is_space] - text_tokens_as_str = [token.text for token in nlp_text if not token.is_space] + text_tokens, text_tokens_as_str = zip( + *( + zip(*((token, token.text) for token in nlp_text if not token.is_space)) + for nlp_text in nlp_text_per_target + ) + ) # in case there is a word at position k that is longer than the max seq len # when converted to wordpiece indexes, _create_word_to_wordpiece_mapping # returns all words up to including k-1 - mapping, text_tokens_as_str = self._create_word_to_wordpiece_mapping( + mapping, text_tokens_as_str = self._batch_create_word_to_wordpiece_mapping( tok_obj, text_tokens_as_str, for_text_with_special_tokens=True, - offset=offset, + offsets=[offset] * len(texts), ) # ..., correspondingly, truncate text_tokens to same length - text_tokens = text_tokens[: len(text_tokens_as_str)] - return mapping, offset, text_tokens_as_str, text_tokens + text_tokens = [ + tt[: len(ttas)] for tt, ttas in zip(text_tokens, text_tokens_as_str) + ] + return mapping, [offset] * len(texts), text_tokens_as_str, text_tokens def _convert_non_null_to_one(self, lst): lst_one = [] @@ -737,11 +830,7 @@ def create_model_input_seqs( will produce an individual target mask for each coref mention and merge them to the target mask of the preferred mention. """ - assert all( - isinstance(arg, str) for arg in (text_left, target_phrase, text_right) - ), "Wrong input types." - - return self.create_model_input_seqs_batch( + return self.batch_create_model_input_seqs( (text_left, target_phrase, text_right), coreferential_targets_for_target_mask=( coreferential_targets_for_target_mask, @@ -750,9 +839,9 @@ def create_model_input_seqs( ) @overload - def create_model_input_seqs_batch( + def batch_create_model_input_seqs( self, - *targets: ..., + targets: ..., coreferential_targets_for_target_mask: ..., batch_size: int = ..., single_target_output: Literal[False] = ..., @@ -760,18 +849,18 @@ def create_model_input_seqs_batch( ... @overload - def create_model_input_seqs_batch( + def batch_create_model_input_seqs( self, - *targets: ..., + targets: ..., coreferential_targets_for_target_mask: ..., batch_size: int = ..., single_target_output: Literal[True] = ..., ) -> Mapping[str, Mapping[str, Union[torch.Tensor, bool]]]: ... - def create_model_input_seqs_batch( + def batch_create_model_input_seqs( self, - *targets: Tuple[str, str, str], + targets: Sequence[Tuple[str, str, str]], coreferential_targets_for_target_mask: Optional[ Sequence[Optional[Iterable[dict]]] ], @@ -792,11 +881,13 @@ def create_model_input_seqs_batch( Multiple target inputs and batching. Output will be a tuple of batches with the created sequences being of shape [batch_size,seq_length]. To output a single target without batch output and with tensors of shape [seq_length], - set squeeze_single_target = True. + set single_target_output = True. """ + # TODO + # - add batch_output:bool to control if output should be in batches or not + # - add batch_size to sub-functions num_targets = len(targets) - num_batches = ceil(num_targets / batch_size) if not coreferential_targets_for_target_mask: coreferential_targets_for_target_mask = [None] * num_targets @@ -815,134 +906,173 @@ def create_model_input_seqs_batch( Consider using create_model_input_seqs() instead.""" ) + lefts = [] + target_phrases = [] + adjusted_target_phrases = [] + full_texts = [] + + for target in targets: + assert all(isinstance(arg, str) for arg in target), "Wrong input types." + + left, target_phrase, _ = target + + lefts.append(left) + + target_phrases.append(target_phrase) + + adjusted_target_phrases.append( + f"What do you think of {target_phrase}?" + if self.is_use_natural_target_phrase_for_spc + else target_phrase + ) + + full_texts.append( + self.create_entire_text(*target, is_return_modified_text_left=False) + ) + out: List[Mapping[str, Mapping[str, Union[torch.Tensor, bool]]]] = [] - for batch_number in range(num_batches): + for batch_start, batch_end in zip( + range(0, num_targets, batch_size), + [*range(batch_size, num_targets, batch_size), None], + ): + # subset all the things we need to get the batch batch_result = {} - batch_start = batch_number * batch_size - batch_end = ( - None - if batch_number == num_batches - 1 - else (batch_number + 1) * batch_size - ) + batch = targets[batch_start:batch_end] + batch_lefts = lefts[batch_start:batch_end] + batch_target_phrases = target_phrases[batch_start:batch_end] + batch_adjusted_target_phrases = adjusted_target_phrases[ + batch_start:batch_end + ] + batch_full_texts = full_texts[batch_start:batch_end] + corefs = coreferential_targets_for_target_mask[batch_start:batch_end] for tok_name, tok_obj in self.tokenizers_name_and_obj.items(): logger.debug(f"{tok_name}") - tokenizer_result = defaultdict(list) - - for (left, target_phrase, right), corefs in zip( - targets[batch_start:batch_end], - coreferential_targets_for_target_mask[batch_start:batch_end], - ): - text = self.create_entire_text( - left, target_phrase, right, is_return_modified_text_left=False - ) - logger.debug(f"'{left}' '{target_phrase}' '{right}'") - - # text - text_ids_with_special_tokens = tok_obj.encode( - text, + # text + text_ids_with_special_tokens_per_target = tok_obj( + text=batch_full_texts, + max_length=self.max_seq_len, + pad_to_max_length=True, + add_special_tokens=True, + truncation="longest_first", + )["input_ids"] + text_then_target_ids_with_special_tokens_dict_per_target = ( + tok_obj.batch_encode_plus( + zip(batch_full_texts, batch_adjusted_target_phrases), max_length=self.max_seq_len, pad_to_max_length=True, add_special_tokens=True, truncation="longest_first", ) - # text SEP target - adjusted_target_phrase = target_phrase - if self.is_use_natural_target_phrase_for_spc: - adjusted_target_phrase = ( - "What do you think of " + target_phrase + "?" - ) - text_then_target_ids_with_special_tokens_dict = tok_obj.encode_plus( - text, - text_pair=adjusted_target_phrase, - max_length=self.max_seq_len, - pad_to_max_length=True, - add_special_tokens=True, - truncation="longest_first", + ) + text_then_target_ids_with_special_tokens_dict_per_target = ( + text_then_target_ids_with_special_tokens_dict_per_target.data + ) + text_then_target_ids_with_special_tokens_per_target = ( + text_then_target_ids_with_special_tokens_dict_per_target[ + "input_ids" + ] + ) + if type(tok_obj) == RobertaTokenizer: + # roberta doesnt have segments, so we produce fake segment ids + # here, and later simply dont pass them + text_then_target_ids_with_special_tokens_segment_ids_per_target = [ + [0 for _ in target_ids] + for target_ids in text_then_target_ids_with_special_tokens_per_target + ] + else: + text_then_target_ids_with_special_tokens_segment_ids_per_target = ( + text_then_target_ids_with_special_tokens_dict_per_target[ + "token_type_ids" + ] ) - text_then_target_ids_with_special_tokens_dict = ( - text_then_target_ids_with_special_tokens_dict.data + # target + target_ids_with_special_tokens_per_target = tok_obj( + batch_target_phrases, + max_length=self.max_seq_len, + pad_to_max_length=True, + add_special_tokens=True, + truncation="longest_first", + )["input_ids"] + + # create target masks + target_mask_seq_for_text_with_special_tokens_per_target = ( + self._batch_create_target_mask( + tok_obj, batch, for_text_with_special_tokens=True ) - text_then_target_ids_with_special_tokens = ( - text_then_target_ids_with_special_tokens_dict["input_ids"] + ) + # create target mask for coreferential targets of given target + coref_target_masks_per_target = ( + self._batch_create_coreferential_target_masks(tok_obj, corefs) + ) + # merge them into the preferred target mask + merged_target_mask_per_target = tuple( + self._merge_coref_target_masks_into_preferred_target_mask( + target_mask, + coref_target_masks, ) - if type(tok_obj) == RobertaTokenizer: - # roberta doesnt have segments, so we produce fake segment ids - # here, and later simply dont pass them - text_then_target_ids_with_special_tokens_segment_ids = [ - 0 - ] * len(text_then_target_ids_with_special_tokens) - else: - text_then_target_ids_with_special_tokens_segment_ids = ( - text_then_target_ids_with_special_tokens_dict[ - "token_type_ids" - ] - ) - # target - target_ids_with_special_tokens = tok_obj.encode( - target_phrase, - max_length=self.max_seq_len, - pad_to_max_length=True, - add_special_tokens=True, - truncation="longest_first", + for target_mask, coref_target_masks in zip( + target_mask_seq_for_text_with_special_tokens_per_target, + coref_target_masks_per_target, ) + ) + target_mask_seq_for_text_with_special_tokens_per_target = ( + merged_target_mask_per_target + ) - # create target masks - target_mask_seq_for_text_with_special_tokens = ( - self._create_target_mask( - tok_obj, - left, - target_phrase, - right, - for_text_with_special_tokens=True, - ) + # create also text ids without max length to see if the one that will be + # used was truncated + text_ids_with_special_tokens_no_max_length_per_target = tok_obj( + full_texts, add_special_tokens=True + )["input_ids"] + text_num_truncated_tokens_per_target = tuple( + len(noml) - len(ml) + for noml, ml in zip( + text_ids_with_special_tokens_no_max_length_per_target, + text_ids_with_special_tokens_per_target, ) - if corefs: - # create target mask for coreferential targets of given target - coref_target_masks = self._create_coreferential_target_masks( - tok_obj, corefs - ) - # merge them into the preferred target mask - merged_target_mask = ( - self._merge_coref_target_masks_into_preferred_target_mask( - target_mask_seq_for_text_with_special_tokens, - coref_target_masks, - ) - ) - target_mask_seq_for_text_with_special_tokens = ( - merged_target_mask - ) + ) - # create also text ids without max length to see if the one that will be - # used was truncated - text_ids_with_special_tokens_no_max_length = tok_obj.encode( - text, add_special_tokens=True - ) - text_num_truncated_tokens = len( - text_ids_with_special_tokens_no_max_length - ) - len(text_ids_with_special_tokens) - - # iterate tokens of full text and look up in dicts - # create mapping from token-based indexes to wordpiece-based indexes - # the function will also remove any tokens for which the word piece index - # would be after max seq len - # under certain circumstances, e.g., when using albert as well as the target - # phrase is close to what would be cut off (but is not at this point), it can happen - # that at a later point in time, i.e., _create_dependency_tree_hop_distances_of_tokens_to_target - # or more specifically _calculate_dep_distance, the target cannot be found in the list of - # tokens anymore, because this list of now cut off. thus, i changed the assert in - # _calculate_dep_distance to throwing an exception that is catched later on - ( - mapping_token2wordpiece, - mapping_token2wordpiece_offset, - text_tokens_as_str, - text_tokens, - ) = self._create_mapping_from_tokenbased_to_wordpiece_based( - text, tok_obj - ) + # iterate tokens of full text and look up in dicts + # create mapping from token-based indexes to wordpiece-based indexes + # the function will also remove any tokens for which the word piece index + # would be after max seq len + # under certain circumstances, e.g., when using albert as well as the target + # phrase is close to what would be cut off (but is not at this point), it can happen + # that at a later point in time, i.e., _create_dependency_tree_hop_distances_of_tokens_to_target + # or more specifically _calculate_dep_distance, the target cannot be found in the list of + # tokens anymore, because this list of now cut off. thus, i changed the assert in + # _calculate_dep_distance to throwing an exception that is catched later on + ( + mapping_token2wordpiece_per_target, + mapping_token2wordpiece_offset_per_target, + text_tokens_as_str_per_target, + text_tokens_per_target, + ) = self._batch_create_mapping_from_tokenbased_to_wordpiece_based( + batch_full_texts, tok_obj + ) + text_stacked_knowledge_source_info_per_target = [] + text_dependency_tree_hop_distances_per_target = [] + text_dependency_matrix_per_target = [] + + for ( + target_phrase, + left, + mapping_token2wordpiece, + mapping_token2wordpiece_offset, + text_tokens_as_str, + text_tokens, + ) in zip( + batch_target_phrases, + batch_lefts, + mapping_token2wordpiece_per_target, + mapping_token2wordpiece_offset_per_target, + text_tokens_as_str_per_target, + text_tokens_per_target, + ): # create additional knowledge source tensors # stack only those knowledge sources that were requested by arguments selected_tensors_knowledge_sources_text = [] @@ -954,15 +1084,14 @@ def create_model_input_seqs_batch( source, ) selected_tensors_knowledge_sources_text.append(text_tensor) - if self.knowledge_sources: - text_stacked_knowledge_source_info = torch.cat( - tuple(selected_tensors_knowledge_sources_text), dim=1 - ) - else: - text_stacked_knowledge_source_info = None + text_stacked_knowledge_source_info_per_target.append( + torch.cat(tuple(selected_tensors_knowledge_sources_text), dim=1) + if self.knowledge_sources + else None + ) # syntax hop distance - text_dependency_tree_hop_distances = ( + text_dependency_tree_hop_distances_per_target.append( self._create_dependency_tree_hop_distances_of_tokens_to_target( text_tokens, len(left), @@ -973,59 +1102,52 @@ def create_model_input_seqs_batch( ) ) - text_dependency_matrix = self._create_dependency_tensor( - text_tokens, - text_tokens_as_str, - mapping_token2wordpiece, - mapping_token2wordpiece_offset, + text_dependency_matrix_per_target.append( + self._create_dependency_tensor( + text_tokens, + text_tokens_as_str, + mapping_token2wordpiece, + mapping_token2wordpiece_offset, + ) ) - # create item with indexes and masks - result = { - FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( - text_ids_with_special_tokens - ), - FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS_TARGET_MASK: torch.FloatTensor( - target_mask_seq_for_text_with_special_tokens - ), - FIELD_IS_OVERFLOW: text_num_truncated_tokens > 0, - FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( - text_then_target_ids_with_special_tokens - ), - FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS_SEGMENT_IDS: torch.LongTensor( - text_then_target_ids_with_special_tokens_segment_ids - ), - FIELD_TARGET_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( - target_ids_with_special_tokens - ), - FIELD_SYNTAX_HOP_DISTANCE_TO_TARGET: text_dependency_tree_hop_distances, - FIELD_SYNTAX_DEPENDENCY_MATRIX: text_dependency_matrix, - } - - # add knowledge source if requested - if self.knowledge_sources: - result[ - FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS_SELECTED_KNOWLEDGE_SOURCES - ] = text_stacked_knowledge_source_info - - # add result to tokenizer result - if single_output: - tokenizer_result.update(result) - else: - for field_name, field_content in result.items(): - tokenizer_result[field_name].append(field_content) - - if not single_output: - # stack tensors in tokenizer result - for field_name, field_content in tokenizer_result.items(): - tokenizer_result[field_name] = ( - torch.stack(field_content) - if isinstance(field_content[0], torch.Tensor) - else tuple(field_content) - ) + # create item with indexes and masks + result = { + FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( + text_ids_with_special_tokens_per_target + ), + FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS_TARGET_MASK: torch.FloatTensor( + target_mask_seq_for_text_with_special_tokens_per_target + ), + FIELD_IS_OVERFLOW: tuple( + text_num > 0 + for text_num in text_num_truncated_tokens_per_target + ), + FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( + text_then_target_ids_with_special_tokens_per_target + ), + FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS_SEGMENT_IDS: torch.LongTensor( + text_then_target_ids_with_special_tokens_segment_ids_per_target + ), + FIELD_TARGET_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( + target_ids_with_special_tokens_per_target + ), + FIELD_SYNTAX_HOP_DISTANCE_TO_TARGET: torch.stack( + text_dependency_tree_hop_distances_per_target + ), + FIELD_SYNTAX_DEPENDENCY_MATRIX: torch.stack( + text_dependency_matrix_per_target + ), + } + + # add knowledge source if requested + if self.knowledge_sources: + result[ + FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS_SELECTED_KNOWLEDGE_SOURCES + ] = torch.stack(text_stacked_knowledge_source_info_per_target) # add tokenizer result to batch result - batch_result[tok_name] = dict(tokenizer_result) + batch_result[tok_name] = result if single_output: # return the single target result if requested @@ -1037,18 +1159,35 @@ def create_model_input_seqs_batch( return out def _create_coreferential_target_masks( - self, tok_obj, coreferential_targets_for_target_mask + self, tok_obj, coreferential_targets_for_target_mask: Optional[Iterable[dict]] + ): + return self._batch_create_coreferential_target_masks( + tok_obj, [coreferential_targets_for_target_mask] + )[0] + + def _batch_create_coreferential_target_masks( + self, + tok_obj, + coreferential_targets_for_target_mask: Sequence[Optional[Iterable[dict]]], ): target_masks = [] - for coref_target in coreferential_targets_for_target_mask: - coref_target_mask = self._create_target_mask( - tok_obj, - self.prepare_left_segment(coref_target["text_left"]), - self.prepare_target_mention(coref_target["mention"]), - "", - for_text_with_special_tokens=True, - ) - target_masks.append(coref_target_mask) + for target in coreferential_targets_for_target_mask: + target_mask = [] + if target: + coref_targets = [ + ( + self.prepare_left_segment(coref_target["text_left"]), + self.prepare_target_mention(coref_target["mention"]), + "", + ) + for coref_target in target + ] + target_mask.append( + self._batch_create_target_mask( + tok_obj, coref_targets, for_text_with_special_tokens=True + ) + ) + target_masks.append(target_mask) return target_masks def _merge_coref_target_masks_into_preferred_target_mask( diff --git a/NewsSentiment/infer.py b/NewsSentiment/infer.py index a1c8c67..0ba83f4 100644 --- a/NewsSentiment/infer.py +++ b/NewsSentiment/infer.py @@ -186,14 +186,14 @@ class probabilities as dictionaries with keys "class_id", "class_label" Must be either one single or multiple component or index based targets.""" return ( - self.infer_batch( + self.batch_infer( component_base if is_component_based else index_base, batch_size=1 )[0] if not is_targets_based - else self.infer_batch(*targets, batch_size=batch_size) + else self.batch_infer(*targets, batch_size=batch_size) ) - def infer_batch( + def batch_infer( self, *targets: Union[Tuple[str, str, str], Tuple[str, int, int]], batch_size: int = 1, @@ -251,8 +251,8 @@ def infer_batch( ) ) - indexed_examples = self.tokenizer.create_model_input_seqs_batch( - *targets_prepared, + indexed_examples = self.tokenizer.batch_create_model_input_seqs( + targets=targets_prepared, coreferential_targets_for_target_mask=None, batch_size=batch_size, single_target_output=False, From 83e0f6e1c545be7b101b959be7b94577d36ecac6 Mon Sep 17 00:00:00 2001 From: "Tilman Hornung (t1h0)" <64684735+t1h0@users.noreply.github.com> Date: Fri, 8 Dec 2023 14:36:31 +0100 Subject: [PATCH 16/32] Move batching outside of infer (add split_and_infer()) --- NewsSentiment/dataset.py | 447 ++++++++++++++++++--------------------- NewsSentiment/infer.py | 134 +++++++----- 2 files changed, 285 insertions(+), 296 deletions(-) diff --git a/NewsSentiment/dataset.py b/NewsSentiment/dataset.py index 6bdf143..4a75ce3 100644 --- a/NewsSentiment/dataset.py +++ b/NewsSentiment/dataset.py @@ -1,5 +1,5 @@ import random -from collections import Counter, defaultdict +from collections import Counter from typing import ( List, Iterable, @@ -229,7 +229,7 @@ def _batch_create_word_to_wordpiece_mapping( if not for_text_with_special_tokens: raise NotImplementedError() - targets_per_target = [] + target_words = [] previous_words_per_target = [] for target in words_per_target: @@ -238,7 +238,7 @@ def _batch_create_word_to_wordpiece_mapping( # words_without_single_whitespace = [word for word in words if word != " "] for word in target: # update left and previous words - targets_per_target.append( + target_words.append( ( " ".join(target_previous_words) + " ", # left word, # target @@ -252,7 +252,7 @@ def _batch_create_word_to_wordpiece_mapping( # produce target masks target_masks = self._batch_create_target_mask( tokenizer=tokenizer, - targets=targets_per_target, + targets=target_words, for_text_with_special_tokens=for_text_with_special_tokens, is_raise_exception_if_target_after_max_seq_len=False, ) @@ -830,22 +830,25 @@ def create_model_input_seqs( will produce an individual target mask for each coref mention and merge them to the target mask of the preferred mention. """ - return self.batch_create_model_input_seqs( + batch_result = self.batch_create_model_input_seqs( (text_left, target_phrase, text_right), coreferential_targets_for_target_mask=( coreferential_targets_for_target_mask, ), - single_target_output=True, ) + return { + tok_name: { + seq_name: seq_result[0] for seq_name, seq_result in tok_result.items() + } + for tok_name, tok_result in batch_result.items() + } @overload def batch_create_model_input_seqs( self, targets: ..., coreferential_targets_for_target_mask: ..., - batch_size: int = ..., - single_target_output: Literal[False] = ..., - ) -> List[Mapping[str, Mapping[str, Union[torch.Tensor, Tuple[bool], bool]]]]: + ) -> Mapping[str, Mapping[str, Union[torch.Tensor, Tuple[bool], bool]]]: ... @overload @@ -853,8 +856,6 @@ def batch_create_model_input_seqs( self, targets: ..., coreferential_targets_for_target_mask: ..., - batch_size: int = ..., - single_target_output: Literal[True] = ..., ) -> Mapping[str, Mapping[str, Union[torch.Tensor, bool]]]: ... @@ -864,12 +865,7 @@ def batch_create_model_input_seqs( coreferential_targets_for_target_mask: Optional[ Sequence[Optional[Iterable[dict]]] ], - batch_size: int = 1, - single_target_output: bool = False, - ) -> Union[ - List[Mapping[str, Mapping[str, Union[torch.Tensor, Tuple[bool], bool]]]], - Mapping[str, Mapping[str, Union[torch.Tensor, bool]]], - ]: + ) -> Mapping[str, Mapping[str, Union[torch.Tensor, Tuple[bool], bool]]]: """ Creates input sequences for given targets. Prior components have processed the jsonl so that coreferential_targets_for_target_mask will contain a list of coreferential @@ -878,15 +874,11 @@ def batch_create_model_input_seqs( the target mask of the preferred mention. NEW - Multiple target inputs and batching. Output will be a tuple of batches with the + Multiple target inputs. Output will be a tuple of batches with the created sequences being of shape [batch_size,seq_length]. To output a single target without batch output and with tensors of shape [seq_length], set single_target_output = True. """ - # TODO - # - add batch_output:bool to control if output should be in batches or not - # - add batch_size to sub-functions - num_targets = len(targets) if not coreferential_targets_for_target_mask: @@ -896,16 +888,6 @@ def batch_create_model_input_seqs( "Number of coreferential_targets_for_target_mask must match number of targets." ) - single_output = False - if single_target_output: - if num_targets == 1: - single_output = True - else: - logger.warning( - """Multiple targets: single_target_output (is True) will be ignored! - Consider using create_model_input_seqs() instead.""" - ) - lefts = [] target_phrases = [] adjusted_target_phrases = [] @@ -930,231 +912,208 @@ def batch_create_model_input_seqs( self.create_entire_text(*target, is_return_modified_text_left=False) ) - out: List[Mapping[str, Mapping[str, Union[torch.Tensor, bool]]]] = [] - - for batch_start, batch_end in zip( - range(0, num_targets, batch_size), - [*range(batch_size, num_targets, batch_size), None], - ): - # subset all the things we need to get the batch - batch_result = {} - batch = targets[batch_start:batch_end] - batch_lefts = lefts[batch_start:batch_end] - batch_target_phrases = target_phrases[batch_start:batch_end] - batch_adjusted_target_phrases = adjusted_target_phrases[ - batch_start:batch_end - ] - batch_full_texts = full_texts[batch_start:batch_end] - corefs = coreferential_targets_for_target_mask[batch_start:batch_end] - - for tok_name, tok_obj in self.tokenizers_name_and_obj.items(): - logger.debug(f"{tok_name}") - - # text - text_ids_with_special_tokens_per_target = tok_obj( - text=batch_full_texts, + out: Mapping[str, Mapping[str, Union[torch.Tensor, bool]]] = {} + + for tok_name, tok_obj in self.tokenizers_name_and_obj.items(): + logger.debug(f"{tok_name}") + + # text + text_ids_with_special_tokens_per_target = tok_obj( + text=full_texts, + max_length=self.max_seq_len, + padding="max_length", + add_special_tokens=True, + truncation="longest_first", + )["input_ids"] + text_then_target_ids_with_special_tokens_dict_per_target = ( + tok_obj.batch_encode_plus( + zip(full_texts, adjusted_target_phrases), max_length=self.max_seq_len, - pad_to_max_length=True, + padding="max_length", add_special_tokens=True, truncation="longest_first", - )["input_ids"] - text_then_target_ids_with_special_tokens_dict_per_target = ( - tok_obj.batch_encode_plus( - zip(batch_full_texts, batch_adjusted_target_phrases), - max_length=self.max_seq_len, - pad_to_max_length=True, - add_special_tokens=True, - truncation="longest_first", - ) - ) - text_then_target_ids_with_special_tokens_dict_per_target = ( - text_then_target_ids_with_special_tokens_dict_per_target.data ) - text_then_target_ids_with_special_tokens_per_target = ( + ) + text_then_target_ids_with_special_tokens_dict_per_target = ( + text_then_target_ids_with_special_tokens_dict_per_target.data + ) + text_then_target_ids_with_special_tokens_per_target = ( + text_then_target_ids_with_special_tokens_dict_per_target["input_ids"] + ) + if type(tok_obj) == RobertaTokenizer: + # roberta doesnt have segments, so we produce fake segment ids + # here, and later simply dont pass them + text_then_target_ids_with_special_tokens_segment_ids_per_target = [ + [0 for _ in target_ids] + for target_ids in text_then_target_ids_with_special_tokens_per_target + ] + else: + text_then_target_ids_with_special_tokens_segment_ids_per_target = ( text_then_target_ids_with_special_tokens_dict_per_target[ - "input_ids" + "token_type_ids" ] ) - if type(tok_obj) == RobertaTokenizer: - # roberta doesnt have segments, so we produce fake segment ids - # here, and later simply dont pass them - text_then_target_ids_with_special_tokens_segment_ids_per_target = [ - [0 for _ in target_ids] - for target_ids in text_then_target_ids_with_special_tokens_per_target - ] - else: - text_then_target_ids_with_special_tokens_segment_ids_per_target = ( - text_then_target_ids_with_special_tokens_dict_per_target[ - "token_type_ids" - ] - ) - # target - target_ids_with_special_tokens_per_target = tok_obj( - batch_target_phrases, - max_length=self.max_seq_len, - pad_to_max_length=True, - add_special_tokens=True, - truncation="longest_first", - )["input_ids"] - - # create target masks - target_mask_seq_for_text_with_special_tokens_per_target = ( - self._batch_create_target_mask( - tok_obj, batch, for_text_with_special_tokens=True - ) + # target + target_ids_with_special_tokens_per_target = tok_obj( + target_phrases, + max_length=self.max_seq_len, + padding="max_length", + add_special_tokens=True, + truncation="longest_first", + )["input_ids"] + + # create target masks + target_mask_seq_for_text_with_special_tokens_per_target = ( + self._batch_create_target_mask( + tok_obj, targets, for_text_with_special_tokens=True ) - # create target mask for coreferential targets of given target - coref_target_masks_per_target = ( - self._batch_create_coreferential_target_masks(tok_obj, corefs) + ) + # create target mask for coreferential targets of given target + coref_target_masks_per_target = ( + self._batch_create_coreferential_target_masks( + tok_obj, coreferential_targets_for_target_mask ) - # merge them into the preferred target mask - merged_target_mask_per_target = tuple( - self._merge_coref_target_masks_into_preferred_target_mask( - target_mask, - coref_target_masks, - ) - for target_mask, coref_target_masks in zip( - target_mask_seq_for_text_with_special_tokens_per_target, - coref_target_masks_per_target, - ) + ) + # merge them into the preferred target mask + merged_target_mask_per_target = tuple( + self._merge_coref_target_masks_into_preferred_target_mask( + target_mask, + coref_target_masks, ) - target_mask_seq_for_text_with_special_tokens_per_target = ( - merged_target_mask_per_target + for target_mask, coref_target_masks in zip( + target_mask_seq_for_text_with_special_tokens_per_target, + coref_target_masks_per_target, ) + ) + target_mask_seq_for_text_with_special_tokens_per_target = ( + merged_target_mask_per_target + ) - # create also text ids without max length to see if the one that will be - # used was truncated - text_ids_with_special_tokens_no_max_length_per_target = tok_obj( - full_texts, add_special_tokens=True - )["input_ids"] - text_num_truncated_tokens_per_target = tuple( - len(noml) - len(ml) - for noml, ml in zip( - text_ids_with_special_tokens_no_max_length_per_target, - text_ids_with_special_tokens_per_target, - ) + # create also text ids without max length to see if the one that will be + # used was truncated + text_ids_with_special_tokens_no_max_length_per_target = tok_obj( + full_texts, add_special_tokens=True + )["input_ids"] + text_num_truncated_tokens_per_target = tuple( + len(noml) - len(ml) + for noml, ml in zip( + text_ids_with_special_tokens_no_max_length_per_target, + text_ids_with_special_tokens_per_target, ) + ) - # iterate tokens of full text and look up in dicts - # create mapping from token-based indexes to wordpiece-based indexes - # the function will also remove any tokens for which the word piece index - # would be after max seq len - # under certain circumstances, e.g., when using albert as well as the target - # phrase is close to what would be cut off (but is not at this point), it can happen - # that at a later point in time, i.e., _create_dependency_tree_hop_distances_of_tokens_to_target - # or more specifically _calculate_dep_distance, the target cannot be found in the list of - # tokens anymore, because this list of now cut off. thus, i changed the assert in - # _calculate_dep_distance to throwing an exception that is catched later on - ( - mapping_token2wordpiece_per_target, - mapping_token2wordpiece_offset_per_target, - text_tokens_as_str_per_target, - text_tokens_per_target, - ) = self._batch_create_mapping_from_tokenbased_to_wordpiece_based( - batch_full_texts, tok_obj - ) + # iterate tokens of full text and look up in dicts + # create mapping from token-based indexes to wordpiece-based indexes + # the function will also remove any tokens for which the word piece index + # would be after max seq len + # under certain circumstances, e.g., when using albert as well as the target + # phrase is close to what would be cut off (but is not at this point), it can happen + # that at a later point in time, i.e., _create_dependency_tree_hop_distances_of_tokens_to_target + # or more specifically _calculate_dep_distance, the target cannot be found in the list of + # tokens anymore, because this list of now cut off. thus, i changed the assert in + # _calculate_dep_distance to throwing an exception that is catched later on + ( + mapping_token2wordpiece_per_target, + mapping_token2wordpiece_offset_per_target, + text_tokens_as_str_per_target, + text_tokens_per_target, + ) = self._batch_create_mapping_from_tokenbased_to_wordpiece_based( + full_texts, tok_obj + ) - text_stacked_knowledge_source_info_per_target = [] - text_dependency_tree_hop_distances_per_target = [] - text_dependency_matrix_per_target = [] - - for ( - target_phrase, - left, - mapping_token2wordpiece, - mapping_token2wordpiece_offset, - text_tokens_as_str, - text_tokens, - ) in zip( - batch_target_phrases, - batch_lefts, - mapping_token2wordpiece_per_target, - mapping_token2wordpiece_offset_per_target, - text_tokens_as_str_per_target, - text_tokens_per_target, - ): - # create additional knowledge source tensors - # stack only those knowledge sources that were requested by arguments - selected_tensors_knowledge_sources_text = [] - for source in self.knowledge_sources: - text_tensor = self._create_knowledge_source_tensor( - text_tokens_as_str, - mapping_token2wordpiece_offset, - mapping_token2wordpiece, - source, - ) - selected_tensors_knowledge_sources_text.append(text_tensor) - text_stacked_knowledge_source_info_per_target.append( - torch.cat(tuple(selected_tensors_knowledge_sources_text), dim=1) - if self.knowledge_sources - else None + text_stacked_knowledge_source_info_per_target = [] + text_dependency_tree_hop_distances_per_target = [] + text_dependency_matrix_per_target = [] + + for ( + target_phrase, + left, + mapping_token2wordpiece, + mapping_token2wordpiece_offset, + text_tokens_as_str, + text_tokens, + ) in zip( + target_phrases, + lefts, + mapping_token2wordpiece_per_target, + mapping_token2wordpiece_offset_per_target, + text_tokens_as_str_per_target, + text_tokens_per_target, + ): + # create additional knowledge source tensors + # stack only those knowledge sources that were requested by arguments + selected_tensors_knowledge_sources_text = [] + for source in self.knowledge_sources: + text_tensor = self._create_knowledge_source_tensor( + text_tokens_as_str, + mapping_token2wordpiece_offset, + mapping_token2wordpiece, + source, ) + selected_tensors_knowledge_sources_text.append(text_tensor) + text_stacked_knowledge_source_info_per_target.append( + torch.cat(tuple(selected_tensors_knowledge_sources_text), dim=1) + if self.knowledge_sources + else None + ) - # syntax hop distance - text_dependency_tree_hop_distances_per_target.append( - self._create_dependency_tree_hop_distances_of_tokens_to_target( - text_tokens, - len(left), - target_phrase, - text_tokens_as_str, - mapping_token2wordpiece, - mapping_token2wordpiece_offset, - ) + # syntax hop distance + text_dependency_tree_hop_distances_per_target.append( + self._create_dependency_tree_hop_distances_of_tokens_to_target( + text_tokens, + len(left), + target_phrase, + text_tokens_as_str, + mapping_token2wordpiece, + mapping_token2wordpiece_offset, ) + ) - text_dependency_matrix_per_target.append( - self._create_dependency_tensor( - text_tokens, - text_tokens_as_str, - mapping_token2wordpiece, - mapping_token2wordpiece_offset, - ) + text_dependency_matrix_per_target.append( + self._create_dependency_tensor( + text_tokens, + text_tokens_as_str, + mapping_token2wordpiece, + mapping_token2wordpiece_offset, ) + ) - # create item with indexes and masks - result = { - FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( - text_ids_with_special_tokens_per_target - ), - FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS_TARGET_MASK: torch.FloatTensor( - target_mask_seq_for_text_with_special_tokens_per_target - ), - FIELD_IS_OVERFLOW: tuple( - text_num > 0 - for text_num in text_num_truncated_tokens_per_target - ), - FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( - text_then_target_ids_with_special_tokens_per_target - ), - FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS_SEGMENT_IDS: torch.LongTensor( - text_then_target_ids_with_special_tokens_segment_ids_per_target - ), - FIELD_TARGET_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( - target_ids_with_special_tokens_per_target - ), - FIELD_SYNTAX_HOP_DISTANCE_TO_TARGET: torch.stack( - text_dependency_tree_hop_distances_per_target - ), - FIELD_SYNTAX_DEPENDENCY_MATRIX: torch.stack( - text_dependency_matrix_per_target - ), - } - - # add knowledge source if requested - if self.knowledge_sources: - result[ - FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS_SELECTED_KNOWLEDGE_SOURCES - ] = torch.stack(text_stacked_knowledge_source_info_per_target) - - # add tokenizer result to batch result - batch_result[tok_name] = result - - if single_output: - # return the single target result if requested - return batch_result - - # add batch result to output - out.append(batch_result) + # create item with indexes and masks + result = { + FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( + text_ids_with_special_tokens_per_target + ), + FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS_TARGET_MASK: torch.FloatTensor( + target_mask_seq_for_text_with_special_tokens_per_target + ), + FIELD_IS_OVERFLOW: tuple( + text_num > 0 for text_num in text_num_truncated_tokens_per_target + ), + FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( + text_then_target_ids_with_special_tokens_per_target + ), + FIELD_TEXT_THEN_TARGET_IDS_WITH_SPECIAL_TOKENS_SEGMENT_IDS: torch.LongTensor( + text_then_target_ids_with_special_tokens_segment_ids_per_target + ), + FIELD_TARGET_IDS_WITH_SPECIAL_TOKENS: torch.LongTensor( + target_ids_with_special_tokens_per_target + ), + FIELD_SYNTAX_HOP_DISTANCE_TO_TARGET: torch.stack( + text_dependency_tree_hop_distances_per_target + ), + FIELD_SYNTAX_DEPENDENCY_MATRIX: torch.stack( + text_dependency_matrix_per_target + ), + } + + # add knowledge source if requested + if self.knowledge_sources: + result[ + FIELD_TEXT_IDS_WITH_SPECIAL_TOKENS_SELECTED_KNOWLEDGE_SOURCES + ] = torch.stack(text_stacked_knowledge_source_info_per_target) + + # add tokenizer result to output + out[tok_name] = result return out @@ -1168,27 +1127,29 @@ def _create_coreferential_target_masks( def _batch_create_coreferential_target_masks( self, tok_obj, - coreferential_targets_for_target_mask: Sequence[Optional[Iterable[dict]]], + coreferential_targets_for_target_mask_per_target: Sequence[ + Optional[Iterable[dict]] + ], ): - target_masks = [] - for target in coreferential_targets_for_target_mask: + target_masks_per_target = [] + for coreferential_targets in coreferential_targets_for_target_mask_per_target: target_mask = [] - if target: + if coreferential_targets: coref_targets = [ ( self.prepare_left_segment(coref_target["text_left"]), self.prepare_target_mention(coref_target["mention"]), "", ) - for coref_target in target + for coref_target in coreferential_targets ] target_mask.append( self._batch_create_target_mask( tok_obj, coref_targets, for_text_with_special_tokens=True ) ) - target_masks.append(target_mask) - return target_masks + target_masks_per_target.append(target_mask) + return target_masks_per_target def _merge_coref_target_masks_into_preferred_target_mask( self, target_mask_seq_for_text_with_special_tokens, coref_target_masks diff --git a/NewsSentiment/infer.py b/NewsSentiment/infer.py index 0ba83f4..21e8fdc 100644 --- a/NewsSentiment/infer.py +++ b/NewsSentiment/infer.py @@ -2,9 +2,10 @@ import torch import torch.nn.functional as F +from math import ceil from jsonlines import jsonlines from tqdm import tqdm -from typing import overload, Any, List, Dict, Union, Sequence, Tuple, Optional, Literal +from typing import overload, Any, List, Dict, Union, Sequence, Tuple, Optional from NewsSentiment.SentimentClasses import SentimentClasses from NewsSentiment.dataset import FXEasyTokenizer @@ -94,7 +95,6 @@ def infer( target_mention_from: None = None, target_mention_to: None = None, targets: None = None, - batch_size: int = ..., ) -> Tuple[Dict[str, Any], ...]: ... @@ -108,7 +108,6 @@ def infer( target_mention_from: int = ..., target_mention_to: int = ..., targets: None = None, - batch_size: int = ..., ) -> Tuple[Dict[str, Any], ...]: ... @@ -122,7 +121,6 @@ def infer( target_mention_from: None = None, target_mention_to: None = None, targets: Sequence[Union[Tuple[str, str, str], Tuple[str, int, int]]] = ..., - batch_size: int = ..., ) -> List[Tuple[Dict[str, Any], ...]]: ... @@ -137,7 +135,6 @@ def infer( targets: Optional[ Sequence[Union[Tuple[str, str, str], Tuple[str, int, int]]] ] = None, - batch_size: int = 1, ) -> Union[Tuple[Dict[str, Any], ...], List[Tuple[Dict[str, Any], ...]]]: """Computes sentiment for a target mention. @@ -157,11 +154,9 @@ def infer( target_mention_to (str | None, optional): End index of the target mention. Defaults to None. targets (Sequence[Tuple[str, str, str] | Tuple[str, int, int]] | None, - optional): Tuples containing text_left,target_mention,text_right - or text,target_mention_from,target_mention_to for multiple targets + optional): Tuples containing (text_left,target_mention,text_right) + or (text,target_mention_from,target_mention_to) for multiple targets (mixed style is possible). Defaults to None. - batch_size (int, optional): Preferred size of batches to compute multiple - targets in. Defaults to 1. Returns: Tuple[Dict[str, Any], ...] | List[Tuple[Dict[str, Any], ...]: Tuple (or @@ -186,40 +181,38 @@ class probabilities as dictionaries with keys "class_id", "class_label" Must be either one single or multiple component or index based targets.""" return ( - self.batch_infer( - component_base if is_component_based else index_base, batch_size=1 - )[0] + self.batch_infer((component_base if is_component_based else index_base,))[0] if not is_targets_based - else self.batch_infer(*targets, batch_size=batch_size) + else self.batch_infer(targets) ) def batch_infer( self, - *targets: Union[Tuple[str, str, str], Tuple[str, int, int]], - batch_size: int = 1, + targets: Sequence[Union[Tuple[str, str, str], Tuple[str, int, int]]], ) -> List[Tuple[Dict[str, Any], ...]]: - """Computes sentiment for multiple targets in batches of batch_size. + """Computes sentiment for a batch of targets. Targets are tuples of text before target, target mention, and text after target. Args: - *targets (Tuple[str,str,str] | Tuple[str,int,int]): Targets to compute - sentiment for. Tuples contain (text before, target mention, text after) - or (text,target mention start index, target mention end index). + targets (Sequence[Tuple[str,str,str] | Tuple[str,int,int]]): Batch of + targets to compute sentiment for. Tuples contain + (text before, target mention, text after) or + (text,target mention start index, target mention end index). Texts before and after the target mention should end with a space (or comma, etc.)), or begin with a space, respectively. - batch_size (int, optional): Preferred size of batches to comppute the targets - in (vectorized computation). Defaults to 1. Returns: List[Tuple[Dict[str,Any], ...]]: List of target classification tuples, - containing class probabilities as dictionaries with keys - "class_id", "class_label" and "class_prob". The order of tuples matches the - order of input targets. + containing class probabilities as dictionaries with keys + "class_id", "class_label" and "class_prob". + The order of tuples matches the order of input targets. """ targets_prepared = [] for target in targets: - assert len(target) == 3, f"{target} is missing one ore more components." + assert ( + len(target) == 3 + ), f"{target} is missing {3-len(target)} component(s)." if all(isinstance(component, str) for component in target): text_left, target_mention, text_right = target @@ -234,7 +227,7 @@ def batch_infer( target_mention = text[target_mention_from:target_mention_to] text_right = text[target_mention_to:] else: - raise TypeError("Wrong input types.") + raise TypeError(f"Wrong input types in {target}.") # assert text_left.endswith(' ') # we cannot handle commas, if we have this # check @@ -254,38 +247,73 @@ def batch_infer( indexed_examples = self.tokenizer.batch_create_model_input_seqs( targets=targets_prepared, coreferential_targets_for_target_mask=None, - batch_size=batch_size, - single_target_output=False, ) - classification_result = [] - - for batch in indexed_examples: - inputs = self.instructor.select_inputs(batch, is_single_item=False) - - outputs = self.model(inputs) - - class_probabilities_per_target = F.softmax(outputs, dim=-1).cpu().tolist() - - classification_result.extend( - tuple( - sorted( - ( - { - "class_id": class_id, - "class_label": self.polarities_inverse[class_id], - "class_prob": class_prob, - } - for class_id, class_prob in enumerate(class_probabilities) - ), - key=lambda x: x["class_prob"], - reverse=True, - ) + inputs = self.instructor.select_inputs(indexed_examples, is_single_item=False) + + outputs = self.model(inputs) + + class_probabilities_per_target = F.softmax(outputs, dim=-1).cpu().tolist() + + return [ + tuple( + sorted( + ( + { + "class_id": class_id, + "class_label": self.polarities_inverse[class_id], + "class_prob": class_prob, + } + for class_id, class_prob in enumerate(class_probabilities) + ), + key=lambda x: x["class_prob"], + reverse=True, ) - for class_probabilities in class_probabilities_per_target ) + for class_probabilities in class_probabilities_per_target + ] - return classification_result + def split_and_infer( + self, + targets: Sequence[Union[Tuple[str, str, str], Tuple[str, int, int]]], + batch_size: int = 32, + ) -> List[Tuple[Dict[str, Any], ...]]: + """Convenience function for splitting input targets into batches, compute + sentiment on each batch (element-wise but vectorized) and return it all in one + output. + + Args: + targets (Sequence[Tuple[str,str,str] | Tuple[str,int,int]]): Targets + to compute sentiment for. Tuples contain + (text before, target mention, text after) or + (text,target mention start index, target mention end index). + Texts before and after the target mention should end with a space + (or comma, etc.)), or begin with a space, respectively. + batch_size (int, optional): Preferred size of each batch. Defaults to 32. + + Returns: + List[Tuple[Dict[str,Any], ...]]: List of target classification tuples, + containing class probabilities as dictionaries with keys + "class_id", "class_label" and "class_prob". + The order of tuples matches the order of input targets. + """ + num_targets = len(targets) + num_batches = ceil(num_targets / batch_size) + + out = [] + + for batch_start, batch_end in tqdm( + zip( + range(0, num_targets, batch_size), + [*range(batch_size, num_targets, batch_size), None], + ), + total=num_batches, + desc="Processing batches", + unit="batch", + ): + out.extend(self.batch_infer(targets[batch_start:batch_end])) + + return out def get_info_for_label(self, classification_result, label): for r in classification_result: From 874fcd28055a471651d0665915db55d30db8eb8d Mon Sep 17 00:00:00 2001 From: "Tilman Hornung (t1h0)" <64684735+t1h0@users.noreply.github.com> Date: Mon, 11 Dec 2023 15:36:42 +0100 Subject: [PATCH 17/32] Bugfix word_to_wordpiece_mapping Function did not return all words up until k-1 when word k's sequence is too long but returned everything up until k --- NewsSentiment/dataset.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/NewsSentiment/dataset.py b/NewsSentiment/dataset.py index 250adc2..2db48a5 100644 --- a/NewsSentiment/dataset.py +++ b/NewsSentiment/dataset.py @@ -192,9 +192,8 @@ def _create_word_to_wordpiece_mapping( # offset word_index by 1 (0th word has index 1) # words_without_single_whitespace = [word for word in words if word != " "] for word_index, word in enumerate(words, start=offset): - # update left and previous words + # update left words left = " ".join(previous_words) + " " - previous_words.append(word) # produce target masks target_mask = self._create_target_mask( @@ -212,6 +211,9 @@ def _create_word_to_wordpiece_mapping( # after the max seq len break + # update previous words + previous_words.append(word) + target_mask = torch.LongTensor(target_mask) target_mask = torch.mul(target_mask, word_index) if mapping is None: From 07ff99a3bff2c6fa500b270ec6295c78b480ff91 Mon Sep 17 00:00:00 2001 From: "Tilman Hornung (t1h0)" <64684735+t1h0@users.noreply.github.com> Date: Mon, 11 Dec 2023 16:24:02 +0100 Subject: [PATCH 18/32] Bugfix in non-batching subfunctions --- NewsSentiment/dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/NewsSentiment/dataset.py b/NewsSentiment/dataset.py index 4a75ce3..fc62291 100644 --- a/NewsSentiment/dataset.py +++ b/NewsSentiment/dataset.py @@ -441,7 +441,7 @@ def _create_target_mask( """ return self._batch_create_target_mask( tokenizer, - [(text_left, target, text_right)], + ((text_left, target, text_right),), for_text_with_special_tokens, is_raise_exception_if_target_after_max_seq_len, )[0] @@ -831,7 +831,7 @@ def create_model_input_seqs( the target mask of the preferred mention. """ batch_result = self.batch_create_model_input_seqs( - (text_left, target_phrase, text_right), + ((text_left, target_phrase, text_right),), coreferential_targets_for_target_mask=( coreferential_targets_for_target_mask, ), @@ -1121,7 +1121,7 @@ def _create_coreferential_target_masks( self, tok_obj, coreferential_targets_for_target_mask: Optional[Iterable[dict]] ): return self._batch_create_coreferential_target_masks( - tok_obj, [coreferential_targets_for_target_mask] + tok_obj, (coreferential_targets_for_target_mask,) )[0] def _batch_create_coreferential_target_masks( From 9d005667935d48e1e33b382c695f01d50cbd6d94 Mon Sep 17 00:00:00 2001 From: "Tilman Hornung (t1h0)" <64684735+t1h0@users.noreply.github.com> Date: Mon, 11 Dec 2023 16:31:01 +0100 Subject: [PATCH 19/32] Fix "toolong" wordpiece mappings --- NewsSentiment/dataset.py | 97 ++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 48 deletions(-) diff --git a/NewsSentiment/dataset.py b/NewsSentiment/dataset.py index fc62291..67ee778 100644 --- a/NewsSentiment/dataset.py +++ b/NewsSentiment/dataset.py @@ -229,25 +229,15 @@ def _batch_create_word_to_wordpiece_mapping( if not for_text_with_special_tokens: raise NotImplementedError() - target_words = [] - previous_words_per_target = [] - - for target in words_per_target: - target_previous_words = [] - - # words_without_single_whitespace = [word for word in words if word != " "] - for word in target: - # update left and previous words - target_words.append( - ( - " ".join(target_previous_words) + " ", # left - word, # target - "", # right - ) - ) - target_previous_words.append(word) - - previous_words_per_target.append(target_previous_words) + target_words = [ + ( + " ".join(target[:word_index]) + " ", # left + word, # target + "", # right + ) + for target in words_per_target + for word_index, word in enumerate(target) + ] # produce target masks target_masks = self._batch_create_target_mask( @@ -257,38 +247,49 @@ def _batch_create_word_to_wordpiece_mapping( is_raise_exception_if_target_after_max_seq_len=False, ) - target_masks = torch.LongTensor(target_masks) - # multiply by word_indices (including offset) - target_masks = torch.mul( - target_masks, - torch.tensor( - tuple( - range(offset, offset + len(wpt)) - for wpt, offset in zip(words_per_target, offsets) - ) - ).reshape((-1, 1)), - ) - # split back into targets - wordpiece_mappings_per_target = target_masks.split( - [len(t) for t in words_per_target], dim=0 + _indexer = 0 + target_masks_per_target = [ + target_masks[_indexer : (_indexer := _indexer + len(wpt))] + for wpt in words_per_target + ] + + # remove "toolong" (word is after the max seq len) + valid_target_masks_per_target = [] + valid_tokens = [] + for target, tokens in zip(target_masks_per_target, words_per_target): + indexer = target.index("toolong") if "toolong" in target else None + valid_target_masks_per_target.append(target[:indexer]) + valid_tokens.append(tokens[:indexer]) + + # convert to tensor + wordpiece_mappings_per_target = torch.LongTensor(valid_target_masks_per_target) + + # multiply by word indices (including offsets) + word_indices_multiplier = torch.tensor( + tuple( + range(offset, offset + len(tmpt)) + for tmpt, offset in zip(valid_target_masks_per_target, offsets) + ) + ).unsqueeze(-1) + wordpiece_mappings_per_target = torch.mul( + wordpiece_mappings_per_target, + word_indices_multiplier, ) - out = [] - for mapping in wordpiece_mappings_per_target: - # make sure there is no overlap - if type(tokenizer) == RobertaTokenizer: - if mapping.prod(0).sum() > 0: - logger.debug( - "overlap when mapping tokens to wordpiece (allow overwriting because" - " Roberta is used)" - ) - else: - assert mapping.prod(0).sum() == 0 - mapping = mapping.sum(0) - assert mapping.shape[0] <= self.max_seq_len - out.append(mapping) - return out, previous_words_per_target + if type(tokenizer) == RobertaTokenizer: + if any(wordpiece_mappings_per_target.prod(1).sum(1) > 0): + logger.debug( + "overlap when mapping tokens to wordpiece (allow overwriting because" + " Roberta is used)" + ) + else: + assert all(wordpiece_mappings_per_target.prod(1).sum(1) == 0) + + mappings = wordpiece_mappings_per_target.sum(1) + assert mappings.shape[1] <= self.max_seq_len + + return mappings, valid_tokens def _calculate_dep_matrix(self, text_tokens, text_tokens_as_str): _check_len_doc = len(text_tokens) From b8d07640276986b5b294a5657c539d30a4bdd22b Mon Sep 17 00:00:00 2001 From: "Tilman Hornung (t1h0)" <64684735+t1h0@users.noreply.github.com> Date: Mon, 11 Dec 2023 17:03:58 +0100 Subject: [PATCH 20/32] Update pypi readme --- READMEpypi.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/READMEpypi.md b/READMEpypi.md index 5c34b00..b1930d0 100644 --- a/READMEpypi.md +++ b/READMEpypi.md @@ -16,7 +16,7 @@ dataset, the model, and its source code can be viewed in our [GitHub repository] # Installation It's super easy, we promise! -You just need a Python 3.7 or Python 3.8 environment. See [here](https://raw.githubusercontent.com/fhamborg/NewsMTSC/main/pythoninfo.md) if you +You just need a Python 3.8 environment. See [here](https://raw.githubusercontent.com/fhamborg/NewsMTSC/main/pythoninfo.md) if you don't have Python or a different version (run `python --version` in a terminal to see your version). Then run: @@ -44,6 +44,32 @@ sentiment = tsc.infer_from_text("" ,"Mark Meadows", "'s coverup of Trump’s cou print(sentiment[0]) ``` +## NEW: Faster classification in batches + +To compute sentiment for batches of targets in a vectorized fashion, you can now feed them all at once into NewsMTSC. + +```python +targets = [ + ("I like ", "Peter", " but I don't like Robert."), + ("", "Mark Meadows", "'s coverup of Trump’s coup attempt is falling apart."), +] + +sentiments = tsc.infer(targets=targets) + +for num_target, result in enumerate(sentiments,1): + print("Target", num_target, result[0]) +``` + +We also provide a convenience method to take care of the batching for you. + +```python +sentiments = tsc.split_and_infer( + targets=targets, batch_size=32 +) +``` + + + # How to identify a person in a sentence? In case your data is not separated as shown in the examples above, i.e., in three segments, you will need to identify one (or more) targets first. From a8b16f57269c145729bd47df738e086bdeb39c6d Mon Sep 17 00:00:00 2001 From: Felix Hamborg Date: Tue, 12 Dec 2023 11:36:48 +0100 Subject: [PATCH 21/32] increase version --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 9e591e8..7e1d00e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = NewsSentiment -version = 1.1.25 +version = 1.2.25 author = Felix Hamborg author_email = felix.hamborg@uni-konstanz.de description = Easy-to-use, high-quality target-dependent sentiment classification for English news articles @@ -28,7 +28,7 @@ classifiers = package_dir = = . packages = find_namespace: -python_requires = >=3.7,<3.9 +python_requires = >=3.8,<3.9 # include_package_data = true install_requires = boto3>=1.19.7 From 901c815e76982aae88d3acd218a3e9524e4c8253 Mon Sep 17 00:00:00 2001 From: Felix Hamborg Date: Tue, 12 Dec 2023 11:44:57 +0100 Subject: [PATCH 22/32] increase version --- READMEpypi.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/READMEpypi.md b/READMEpypi.md index b1930d0..522ad23 100644 --- a/READMEpypi.md +++ b/READMEpypi.md @@ -44,9 +44,10 @@ sentiment = tsc.infer_from_text("" ,"Mark Meadows", "'s coverup of Trump’s cou print(sentiment[0]) ``` -## NEW: Faster classification in batches +## Faster classification in batches (faster) -To compute sentiment for batches of targets in a vectorized fashion, you can now feed them all at once into NewsMTSC. +To compute sentiment for batches of targets in a vectorized fashion, you can also feed them all at once into NewsMTSC. This +is much faster than calling `infer_from_text` multiple times. ```python targets = [ @@ -56,8 +57,8 @@ targets = [ sentiments = tsc.infer(targets=targets) -for num_target, result in enumerate(sentiments,1): - print("Target", num_target, result[0]) +for num_target, result in enumerate(sentiments): + print("Target: ", num_target, result[0]) ``` We also provide a convenience method to take care of the batching for you. From ee2e376a489329a26eeaa7eccf9bdc5195effb7d Mon Sep 17 00:00:00 2001 From: "Tilman Hornung (t1h0)" <64684735+t1h0@users.noreply.github.com> Date: Tue, 12 Dec 2023 13:10:31 +0100 Subject: [PATCH 23/32] Merge split_and_infer() into infer() --- NewsSentiment/infer.py | 86 +++++++++++++++++++----------------------- READMEpypi.md | 12 +----- 2 files changed, 40 insertions(+), 58 deletions(-) diff --git a/NewsSentiment/infer.py b/NewsSentiment/infer.py index 21e8fdc..db71d23 100644 --- a/NewsSentiment/infer.py +++ b/NewsSentiment/infer.py @@ -95,6 +95,8 @@ def infer( target_mention_from: None = None, target_mention_to: None = None, targets: None = None, + batch_size: int = ..., + disable_tqdm: bool = ..., ) -> Tuple[Dict[str, Any], ...]: ... @@ -108,6 +110,8 @@ def infer( target_mention_from: int = ..., target_mention_to: int = ..., targets: None = None, + batch_size: int = ..., + disable_tqdm: bool = ..., ) -> Tuple[Dict[str, Any], ...]: ... @@ -121,6 +125,8 @@ def infer( target_mention_from: None = None, target_mention_to: None = None, targets: Sequence[Union[Tuple[str, str, str], Tuple[str, int, int]]] = ..., + batch_size: int = ..., + disable_tqdm: bool = ..., ) -> List[Tuple[Dict[str, Any], ...]]: ... @@ -135,11 +141,14 @@ def infer( targets: Optional[ Sequence[Union[Tuple[str, str, str], Tuple[str, int, int]]] ] = None, + batch_size: int = 1, + disable_tqdm: bool = False, ) -> Union[Tuple[Dict[str, Any], ...], List[Tuple[Dict[str, Any], ...]]]: - """Computes sentiment for a target mention. + """Computes sentiment for one or more targets. Additionally splits multiple + targets into batches of batch_size and processes them in a vectorized fashion. - Note that the text before and after should end with a space (or comma, etc.)), - or begin with a space, respectively. + Note that the text before and after the target should end with a space + (or comma, etc.)), or begin with a space, respectively. Args: text_left (str | None, optional): Text before the target mention. @@ -157,6 +166,10 @@ def infer( optional): Tuples containing (text_left,target_mention,text_right) or (text,target_mention_from,target_mention_to) for multiple targets (mixed style is possible). Defaults to None. + batch_size (int, optional): Preferred size of each batch if using multiple + targets. Defaults to 1. + disable_tqdm (bool, optional): Disables the tqdm progress bar that shows + progress in batch processing. Defaults to False. Returns: Tuple[Dict[str, Any], ...] | List[Tuple[Dict[str, Any], ...]: Tuple (or @@ -180,12 +193,31 @@ class probabilities as dictionaries with keys "class_id", "class_label" ), """Wrong input types or too many inputs! Must be either one single or multiple component or index based targets.""" - return ( - self.batch_infer((component_base if is_component_based else index_base,))[0] + targets = ( + (component_base if is_component_based else index_base,) if not is_targets_based - else self.batch_infer(targets) + else targets ) + num_targets = len(targets) + num_batches = ceil(num_targets / batch_size) + + out = [] + + for batch_start, batch_end in tqdm( + zip( + range(0, num_targets, batch_size), + (*range(batch_size, num_targets, batch_size), None), + ), + total=num_batches, + desc="Processing batches", + unit="batch", + disable=disable_tqdm, + ): + out.extend(self.batch_infer(targets[batch_start:batch_end])) + + return out[0] if not is_targets_based else out + def batch_infer( self, targets: Sequence[Union[Tuple[str, str, str], Tuple[str, int, int]]], @@ -273,48 +305,6 @@ def batch_infer( for class_probabilities in class_probabilities_per_target ] - def split_and_infer( - self, - targets: Sequence[Union[Tuple[str, str, str], Tuple[str, int, int]]], - batch_size: int = 32, - ) -> List[Tuple[Dict[str, Any], ...]]: - """Convenience function for splitting input targets into batches, compute - sentiment on each batch (element-wise but vectorized) and return it all in one - output. - - Args: - targets (Sequence[Tuple[str,str,str] | Tuple[str,int,int]]): Targets - to compute sentiment for. Tuples contain - (text before, target mention, text after) or - (text,target mention start index, target mention end index). - Texts before and after the target mention should end with a space - (or comma, etc.)), or begin with a space, respectively. - batch_size (int, optional): Preferred size of each batch. Defaults to 32. - - Returns: - List[Tuple[Dict[str,Any], ...]]: List of target classification tuples, - containing class probabilities as dictionaries with keys - "class_id", "class_label" and "class_prob". - The order of tuples matches the order of input targets. - """ - num_targets = len(targets) - num_batches = ceil(num_targets / batch_size) - - out = [] - - for batch_start, batch_end in tqdm( - zip( - range(0, num_targets, batch_size), - [*range(batch_size, num_targets, batch_size), None], - ), - total=num_batches, - desc="Processing batches", - unit="batch", - ): - out.extend(self.batch_infer(targets[batch_start:batch_end])) - - return out - def get_info_for_label(self, classification_result, label): for r in classification_result: if r["class_label"] == label: diff --git a/READMEpypi.md b/READMEpypi.md index 522ad23..dc04836 100644 --- a/READMEpypi.md +++ b/READMEpypi.md @@ -55,21 +55,13 @@ targets = [ ("", "Mark Meadows", "'s coverup of Trump’s coup attempt is falling apart."), ] -sentiments = tsc.infer(targets=targets) +# adjust batch_size to your needs (e.g. 32 or 64 for bigger data) +sentiments = tsc.infer(targets=targets, batch_size=2) for num_target, result in enumerate(sentiments): print("Target: ", num_target, result[0]) ``` -We also provide a convenience method to take care of the batching for you. - -```python -sentiments = tsc.split_and_infer( - targets=targets, batch_size=32 -) -``` - - # How to identify a person in a sentence? From 0c98fdc58332bc9b23831685deadf12e81a1f690 Mon Sep 17 00:00:00 2001 From: Felix Hamborg Date: Tue, 12 Dec 2023 13:19:21 +0100 Subject: [PATCH 24/32] increase version --- READMEpypi.md | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/READMEpypi.md b/READMEpypi.md index dc04836..220de59 100644 --- a/READMEpypi.md +++ b/READMEpypi.md @@ -37,37 +37,36 @@ Since this is a one-time process, future use of NewsSentiment will be much faste from NewsSentiment import TargetSentimentClassifier tsc = TargetSentimentClassifier() -sentiment = tsc.infer_from_text("I like " ,"Peter", " but I don't like Robert.") -print(sentiment[0]) - -sentiment = tsc.infer_from_text("" ,"Mark Meadows", "'s coverup of Trump’s coup attempt is falling apart.") -print(sentiment[0]) -``` - -## Faster classification in batches (faster) - -To compute sentiment for batches of targets in a vectorized fashion, you can also feed them all at once into NewsMTSC. This -is much faster than calling `infer_from_text` multiple times. - -```python -targets = [ +data = [ ("I like ", "Peter", " but I don't like Robert."), ("", "Mark Meadows", "'s coverup of Trump’s coup attempt is falling apart."), ] -# adjust batch_size to your needs (e.g. 32 or 64 for bigger data) -sentiments = tsc.infer(targets=targets, batch_size=2) +sentiments = tsc.infer(targets=data) -for num_target, result in enumerate(sentiments): - print("Target: ", num_target, result[0]) +for i, result in enumerate(sentiments): + print("Sentiment: ", i, result[0]) ``` +This method will internally split the data into batches of size 16 for increased speed. You can adjust the +batch size using the `batch_size` parameter, e.g., `batch_size=32`. + +Alternatively, you can also use the `infer_from_text` method to infer sentiment for a single target: + +```python +sentiment = tsc.infer_from_text("I like " ,"Peter", " but I don't like Robert.") +print(sentiment[0]) +``` # How to identify a person in a sentence? In case your data is not separated as shown in the examples above, i.e., in three segments, you will need to identify one (or more) targets first. How this is done best depends on your project and analysis task but you may, for example, use NER. This [example](https://github.com/fhamborg/NewsMTSC/issues/30#issuecomment-1700645679) shows a simple way of doing so. +# Acknowledgements + +Thanks to [Tilman Hornung](https://github.com/t1h0) for adding the batching functionality and various other improvements. + # How to cite If you use the dataset or model, please cite our [paper](https://www.aclweb.org/anthology/2021.eacl-main.142/) ([PDF](https://www.aclweb.org/anthology/2021.eacl-main.142.pdf)): From bd8737376cabfcbedbe5c6c159f425c3f498750b Mon Sep 17 00:00:00 2001 From: "Tilman Hornung (t1h0)" <64684735+t1h0@users.noreply.github.com> Date: Tue, 12 Dec 2023 18:42:40 +0100 Subject: [PATCH 25/32] Bugfix targets need to be handled separately (not in one tensor) because their number of words are different thus have different tensor sizes of [num_words,seq_len] --- NewsSentiment/dataset.py | 45 ++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/NewsSentiment/dataset.py b/NewsSentiment/dataset.py index 67ee778..821124c 100644 --- a/NewsSentiment/dataset.py +++ b/NewsSentiment/dataset.py @@ -262,32 +262,31 @@ def _batch_create_word_to_wordpiece_mapping( valid_target_masks_per_target.append(target[:indexer]) valid_tokens.append(tokens[:indexer]) - # convert to tensor - wordpiece_mappings_per_target = torch.LongTensor(valid_target_masks_per_target) - - # multiply by word indices (including offsets) - word_indices_multiplier = torch.tensor( - tuple( - range(offset, offset + len(tmpt)) - for tmpt, offset in zip(valid_target_masks_per_target, offsets) + # convert to tensors + mappings = [] + + for target, offset in zip(valid_target_masks_per_target, offsets): + target_tensor = torch.LongTensor(target) + + word_indices_multiplier = torch.tensor( + range(offset, offset + len(target_tensor)) ) - ).unsqueeze(-1) - wordpiece_mappings_per_target = torch.mul( - wordpiece_mappings_per_target, - word_indices_multiplier, - ) + target_tensor = target_tensor.mul(word_indices_multiplier.unsqueeze(-1)) - if type(tokenizer) == RobertaTokenizer: - if any(wordpiece_mappings_per_target.prod(1).sum(1) > 0): - logger.debug( - "overlap when mapping tokens to wordpiece (allow overwriting because" - " Roberta is used)" - ) - else: - assert all(wordpiece_mappings_per_target.prod(1).sum(1) == 0) + if type(tokenizer) == RobertaTokenizer: + if target_tensor.prod(0).sum(0) > 0: + logger.debug( + "overlap when mapping tokens to wordpiece (allow overwriting because" + " Roberta is used)" + ) + else: + assert target_tensor.prod(0).sum(0) == 0 + + target_tensor = target_tensor.sum(0) + + assert target_tensor.size(0) <= self.max_seq_len - mappings = wordpiece_mappings_per_target.sum(1) - assert mappings.shape[1] <= self.max_seq_len + mappings.append(target_tensor) return mappings, valid_tokens From 7dcb999f3fed5fa64883e9fa257ff395e3406ed6 Mon Sep 17 00:00:00 2001 From: "Tilman Hornung (t1h0)" <64684735+t1h0@users.noreply.github.com> Date: Tue, 12 Dec 2023 18:42:49 +0100 Subject: [PATCH 26/32] Increase version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 37fd181..71831ac 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = NewsSentiment -version = 1.2.25 +version = 1.2.26 author = Felix Hamborg author_email = felix.hamborg@uni-konstanz.de description = Easy-to-use, high-quality target-dependent sentiment classification for English news articles From ac57195a02d148f4dedc59e4f894a821d7a84dcd Mon Sep 17 00:00:00 2001 From: "Tilman Hornung (t1h0)" <64684735+t1h0@users.noreply.github.com> Date: Thu, 14 Dec 2023 15:08:18 +0100 Subject: [PATCH 27/32] Suppress transformers warning "Some weights.." --- NewsSentiment/train.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/NewsSentiment/train.py b/NewsSentiment/train.py index db5629e..22e3f6a 100644 --- a/NewsSentiment/train.py +++ b/NewsSentiment/train.py @@ -6,6 +6,7 @@ import time from collections import Counter from typing import Iterable +import logging import numpy import torch @@ -261,9 +262,19 @@ def create_transformer_model( self.transformer_tokenizers[ pretrained_weights_name ] = tokenizer_class.from_pretrained(model_path) + + # supress the transformers warning + # "Some weights of the model checkpoint..were not used.." + transformers_logger = logging.getLogger('transformers.modeling_utils') + transformers_logger_level = transformers_logger.getEffectiveLevel() + transformers_logger.setLevel(logging.ERROR) + self.transformer_models[ pretrained_weights_name ] = model_class.from_pretrained(model_path, output_hidden_states=True) + + # reset transformers logging level + transformers_logger.setLevel(transformers_logger_level) def _reset_params_of_own_model(self): for child in self.own_model.children(): From be52ac9bc0c29cf954121b36ae3ba008ac848c30 Mon Sep 17 00:00:00 2001 From: Felix Hamborg Date: Thu, 14 Dec 2023 15:11:35 +0100 Subject: [PATCH 28/32] supress warnings --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 71831ac..d697822 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = NewsSentiment -version = 1.2.26 +version = 1.2.27 author = Felix Hamborg author_email = felix.hamborg@uni-konstanz.de description = Easy-to-use, high-quality target-dependent sentiment classification for English news articles From fec8b7d6017d209395507480b6765b4ee9bd7b8e Mon Sep 17 00:00:00 2001 From: "Tilman Hornung (t1h0)" <64684735+t1h0@users.noreply.github.com> Date: Mon, 18 Dec 2023 22:44:04 +0100 Subject: [PATCH 29/32] Raise torch upper bound to <2.1 --- NewsSentiment/train.py | 2 +- setup.cfg | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/NewsSentiment/train.py b/NewsSentiment/train.py index 22e3f6a..0c7532b 100644 --- a/NewsSentiment/train.py +++ b/NewsSentiment/train.py @@ -390,7 +390,7 @@ def _train(self, criterion, optimizer, train_data_loader, dev_data_loader): for i_batch, sample_batched in enumerate(train_data_loader): global_step += 1 # clear gradient accumulators - optimizer.zero_grad() + optimizer.zero_grad(set_to_none = False) # select only relevant fields inputs = self.select_inputs(sample_batched) targets = sample_batched["polarity"].to(self.opt.device) diff --git a/setup.cfg b/setup.cfg index d697822..4e878f2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,8 +45,8 @@ install_requires = spacy>=3.2 tabulate>=0.8.9 tqdm>=4.62.3 - transformers>=4.17.0,<=4.24.0 - torch>=1.12.0,<1.14.0 + transformers>=4.17,<=4.24 + torch>=1.12,<2.1 [options.packages.find] where = . From f44445f2c228c6887816d3b7483ce494851b44c6 Mon Sep 17 00:00:00 2001 From: "Tilman Hornung (t1h0)" <64684735+t1h0@users.noreply.github.com> Date: Mon, 18 Dec 2023 22:44:28 +0100 Subject: [PATCH 30/32] Raise python upper bound to <3.12 --- README.md | 2 ++ pythoninfo.md | 9 +++++---- setup.cfg | 5 ++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 897b2f9..d712e27 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ virtualenv -ppython3.8 --setuptools 45 venv source venv/bin/activate ``` +Note: We recommend Python 3.8, however we have successfully tested NewsMTSC with Python version >=3.8, <3.12. + **2. Setup NewsMTSC:** ```bash git clone git@github.com:fhamborg/NewsMTSC.git diff --git a/pythoninfo.md b/pythoninfo.md index deaafc4..2052c39 100644 --- a/pythoninfo.md +++ b/pythoninfo.md @@ -1,7 +1,8 @@ -This step is optional if you have Python 3.8 installed (run `python --version` -in a terminal and check the version that is printed). If you don't have Python 3.8, we -recommend using Anaconda for setting up requirements because it is very easy (but any way -of installing Python 3.8 is fine). If you do not have Anaconda yet, follow their +This step is optional if you have Python >=3.8, <3.12 installed (run `python --version` +in a terminal and check the version that is printed; we recommend 3.8). If you don't +have Python (in the correct version), we recommend using Anaconda for setting up +requirements because it is very easy (but any way of installing is fine). +If you do not have Anaconda yet, follow their [installation instructions](https://docs.anaconda.com/anaconda/install/). After installing Anaconda, to set up a Python 3.8 environment (in case you don't have one diff --git a/setup.cfg b/setup.cfg index 4e878f2..17f57a8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,6 +16,9 @@ classifiers = Operating System :: OS Independent Programming Language :: Python :: 3 Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Intended Audience :: Developers Intended Audience :: Science/Research Topic :: Scientific/Engineering @@ -27,7 +30,7 @@ classifiers = package_dir = = . packages = find_namespace: -python_requires = ==3.8.* +python_requires = >=3.8, <3.12 # include_package_data = true install_requires = boto3>=1.19.7 From a7bea7bc7265b4bfe4236e98965a47c767acbb30 Mon Sep 17 00:00:00 2001 From: Felix Hamborg Date: Wed, 20 Dec 2023 17:15:58 +0100 Subject: [PATCH 31/32] incr version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 17f57a8..88acfcd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = NewsSentiment -version = 1.2.27 +version = 1.2.28 author = Felix Hamborg author_email = felix.hamborg@uni-konstanz.de description = Easy-to-use, high-quality target-dependent sentiment classification for English news articles From b615b7f3e507cdb11f7bde4d0380fe0f483025cb Mon Sep 17 00:00:00 2001 From: Sifat Anindho <102836340+sifatanindho@users.noreply.github.com> Date: Tue, 12 Nov 2024 17:13:14 -0700 Subject: [PATCH 32/32] Update readme.md I think you guys misspelled `devtest_rw.jsonl` as `devtest_mt.jsonl` in the description for the real-world distribution split. Not a big issue, just something I noticed. --- .../controller_data/datasets/NewsMTSC-dataset/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NewsSentiment/controller_data/datasets/NewsMTSC-dataset/readme.md b/NewsSentiment/controller_data/datasets/NewsMTSC-dataset/readme.md index afba64f..c5ff2ad 100644 --- a/NewsSentiment/controller_data/datasets/NewsMTSC-dataset/readme.md +++ b/NewsSentiment/controller_data/datasets/NewsMTSC-dataset/readme.md @@ -9,7 +9,7 @@ The dataset consists of three splits. In practical terms, we suggest to use the * `train.jsonl` - For **training**. * `devtest_mt.jsonl` - To evaluate a model's classification performance only on sentences that contain **at least two target mentions**. Note that the mentions were extracted to refer to different persons but in a few cases might indeed refer to the same person since we extracted them automatically. -* `devtest_mt.jsonl` - To evaluate a model's classification performance on a "**real-world**" set of sentences, i.e., the set was created with the objective to resemble real-world distribution as to sentiment and other factors mentioned in the paper. +* `devtest_rw.jsonl` - To evaluate a model's classification performance on a "**real-world**" set of sentences, i.e., the set was created with the objective to resemble real-world distribution as to sentiment and other factors mentioned in the paper. ### Format