diff --git a/.github/workflows/deploy_react.yml b/.github/workflows/deploy_react.yml deleted file mode 100644 index 31a540d..0000000 --- a/.github/workflows/deploy_react.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Deploy React App to GitHub Pages - -on: - push: - branches: - - "gh-deploy" - -jobs: - deploy: - runs-on: ubuntu-latest - - strategy: - matrix: - node-version: [15.x] - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Install and build - run: cd web-app && npm install && npm run build - - - name: Deploy - uses: JamesIves/github-pages-deploy-action@v4 - with: - folder: web-app/build diff --git a/.github/workflows/test_build_deploy.yml b/.github/workflows/test_build_deploy.yml new file mode 100644 index 0000000..682f478 --- /dev/null +++ b/.github/workflows/test_build_deploy.yml @@ -0,0 +1,65 @@ +name: Test, Build and Deploy to GitHub Pages + +on: + push: + branches: + - "main" + +jobs: + build_and_test_data: + name: Build and test data + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.7"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + cd data + pip install -r requirements.txt + - name: Test with coverage and pytest + run: | + cd data && cd src && cd tests && coverage run -m pytest + - name: Create coverage report + run: | + cd data && cd src && cd tests && coverage report + - name: Create HTML coverage report + run: | + cd data && cd src && cd tests && coverage html + - name: Store test coverage report artifact + uses: actions/upload-artifact@v2 + with: + name: data-coverage-report + path: data/src/tests/htmlcov + + # test_react: + # name: Test React + # TODO: + + deploy_react: + name: Deploy React app + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [15.x] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Install and build + run: cd web-app && npm install && npm run build + + - name: Deploy + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: web-app/build diff --git a/.gitignore b/.gitignore index c53e8ee..a44e475 100644 --- a/.gitignore +++ b/.gitignore @@ -6,15 +6,15 @@ data/models/* data/temp data/temp/* data/.pytest_cache -data/tests/.pytest_cache +data/src/tests/.pytest_cache data/src/__pycache__ data/src/__pycache__/* data/src/.pytest_cache data/src/.pytest_cache/* data/.coverage data/src/.coverage -data/tests/__pycache__ -data/tests/__pycache__/* -data/tests/.coverage -data/tests/temp_test_files -data/tests/temp_test_files/* \ No newline at end of file +data/src/tests/__pycache__ +data/src/tests/__pycache__/* +data/src/tests/.coverage +data/src/tests/temp_test_files +data/src/tests/temp_test_files/* \ No newline at end of file diff --git a/README.md b/README.md index 017629c..f4fea15 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Newspaper Topic Modelling 📰 🔍 -NLP topic modelling of UK newspapers, with analysis of topics over time, as well as sentiment analysis of polarity and subjectivity of language used. +NLP topic modelling of UK newspapers, with analysis of topics over time, as well as sentiment analysis of polarity and subjectivity of language used. Python data analysis and React JSX website presenting that analysis, which is live here: [https://czboop.github.io/Newspaper-Topic-Modelling/](https://czboop.github.io/Newspaper-Topic-Modelling/) ## Project Summary This project uses several techniques within natural language processing to explore seven of the top newspapers in the UK. Data analysed for all sources covered the period from just before the start of the COVID-19 pandemic (late November 2019), until the start of 2023 (early January). The newspapers analysed were: @@ -114,6 +114,26 @@ Some of the key repository contents: * [Pytest](https://docs.pytest.org/en/7.3.x/) - to run tests from the command line * [Pandas testing](https://pandas.pydata.org/docs/reference/testing.html) - to assert dataframe equality -## How to Install and Run +## How to Install and Run the Data Analysis +To get set up to run the Python/data portion of the project: -## How to Use +* If Python is not installed, install it from [this link](https://www.python.org/downloads/). +* Clone this repository, then navigate to the directory it is in. +* Set up a virtual environment using: +```$ python -m venv ``` +* Activate the virtual environment. For Windows, this is done using: +```$ \Scripts\activate.bat``` +[This link](https://docs.python.org/3/library/venv.html) shows how to do this for other operating systems. +* Install dependencies using: +```$ pip install -r requirements.txt ``` +* After navigating to the directory with the desired file, one of the Python files can be run using: +```$ python .py``` + +**NOTE: At least one of the dependencies may have issues running with the latest version of Python. Downgrading to version 3.7 in your virtual environment may be required.** This can be done by downloading Python 3.7, and creating the virtual environment specifying that version: ```$ python3.7 -m venv ``` + +The scripts are made up of classes/objects that take in as part of their constructor, a path to a directory that is expected to contain .csv files with the data to be analysed. This should be updated to reflect wherever your local data files are stored. The default can be updated in the Python files that define the classes, or a different path can be given when creating an instance of the class. + +Also, the scripts make assumptions about the columns that should be present in the data ('headline', 'date', and 'url'), that should likely be updated to match any new data that they are being run on. + +## How to Use the Web App +Check out the React website hosted on GitHub Pages, which presents many of the findings of the topic modelling and sentiment analysis, as well as data visualistions. [Link to website](https://czboop.github.io/Newspaper-Topic-Modelling/) diff --git a/data/__init__.py b/data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/data/conftest.py b/data/conftest.py new file mode 100644 index 0000000..e69de29 diff --git a/data/requirements.txt b/data/requirements.txt index e1094b5..1d514d7 100644 --- a/data/requirements.txt +++ b/data/requirements.txt @@ -68,7 +68,6 @@ pytest==7.3.0 python-dateutil==2.8.2 python-docx==0.8.11 pytz==2022.7 -pywin32==305 PyYAML==5.4.1 regex==2022.10.31 requests==2.28.1 diff --git a/data/src/data_processor.py b/data/src/data_processor.py index a9eeaea..1d249c4 100644 --- a/data/src/data_processor.py +++ b/data/src/data_processor.py @@ -3,6 +3,7 @@ import datetime from datetime import datetime as dt from spacy.lang.en.stop_words import STOP_WORDS +import os # read concat and select data for topic modelling class DataProcessor: @@ -36,7 +37,7 @@ def read_and_concat_data_files(self): # creating pandas dataframe from each csv file df = pd.read_csv(file_, usecols = self.cols) # setting source column from path - first part of file name separated by underscores e.g. dir1/dir2/dir3/sourcename_date1_to_date2.csv - df['source'] = file_.split("\\")[-1].split("_")[0] + df['source'] = os.path.normpath(file_).split(os.path.sep)[-1].split("_")[0] df_list.append(df) # combining if multiple files, returning original df if one file, raising exception if no files or error diff --git a/data/src/general_analyser.py b/data/src/general_analyser.py index 80bb221..7649804 100644 --- a/data/src/general_analyser.py +++ b/data/src/general_analyser.py @@ -1,4 +1,4 @@ -from src.data_processor import DataProcessor +from data_processor import DataProcessor import plotly.express as px import pandas as pd from pathlib import Path @@ -114,4 +114,7 @@ def run(self): self.visualise_percentages(self.compare_ratio_of_docs()[2]) number_by_source, number_total = self.compare_num_of_docs_over_time() self.visualise_number_over_time(number_by_source, source_name = "All Sources") - self.visualise_number_over_time(number_total, single = True, source_name = "Combined Sources") \ No newline at end of file + self.visualise_number_over_time(number_total, single = True, source_name = "Combined Sources") + +if __name__ == "__main__": + print("ran") \ No newline at end of file diff --git a/data/src/multi_source_modeller.py b/data/src/multi_source_modeller.py index d879443..f16731a 100644 --- a/data/src/multi_source_modeller.py +++ b/data/src/multi_source_modeller.py @@ -1,6 +1,6 @@ from bertopic import BERTopic import datetime -from src.topic_modeller import TopicModeller +from topic_modeller import TopicModeller from pathlib import Path # modelling all the different news sources or multiple at once diff --git a/data/src/multi_source_sentiments.py b/data/src/multi_source_sentiments.py index b59e448..f9a2337 100644 --- a/data/src/multi_source_sentiments.py +++ b/data/src/multi_source_sentiments.py @@ -1,5 +1,5 @@ -from src.sentiment_analyser import SentimentAnalyser -from src.data_processor import DataProcessor +from sentiment_analyser import SentimentAnalyser +from data_processor import DataProcessor import datetime # getting sentiment (polarity and subjectivity) info and graphs for all sources diff --git a/data/src/sentiment_analyser.py b/data/src/sentiment_analyser.py index 7c0d811..974db4b 100644 --- a/data/src/sentiment_analyser.py +++ b/data/src/sentiment_analyser.py @@ -1,4 +1,4 @@ -from src.data_processor import DataProcessor +from data_processor import DataProcessor import spacy from spacytextblob.spacytextblob import SpacyTextBlob import pandas as pd diff --git a/data/src/tests/__init__.py b/data/src/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/data/tests/test_data_processor.py b/data/src/tests/test_data_processor.py similarity index 95% rename from data/tests/test_data_processor.py rename to data/src/tests/test_data_processor.py index 38a4d41..ad2f2fc 100644 --- a/data/tests/test_data_processor.py +++ b/data/src/tests/test_data_processor.py @@ -2,8 +2,7 @@ from pathlib import Path import sys sys.path.append(f"{Path(__file__).parent.parent}") -import src -from src.data_processor import DataProcessor +from data_processor import DataProcessor import os from pathlib import Path import pandas as pd @@ -36,12 +35,12 @@ def test_select_data_files(self): data_processor = DataProcessor(f'./{self.test_dir_name}', ['headline', 'url'], selector= 'test*.csv') # when - we call the select data files method of the data processor - actual = data_processor.select_data_files() + actual = map(lambda x: os.path.normpath(x), data_processor.select_data_files()) # then - an array of the five test files is returned, and the data processor has a new property called files - expected = [f'./{self.test_dir_name}\\test_0.csv', f'./{self.test_dir_name}\\test_1.csv', f'./{self.test_dir_name}\\test_2.csv', f'./{self.test_dir_name}\\test_3.csv', f'./{self.test_dir_name}\\test_4.csv'] + expected = map(lambda x: os.path.normpath(x), [f'./{self.test_dir_name}/test_0.csv', f'./{self.test_dir_name}/test_1.csv', f'./{self.test_dir_name}/test_2.csv', f'./{self.test_dir_name}/test_3.csv', f'./{self.test_dir_name}/test_4.csv']) - self.assertEqual(actual, expected) + self.assertEqual(sorted(actual), sorted(expected)) self.assertTrue(hasattr(data_processor, 'files')) def test_read_and_concat_data_files(self): @@ -100,7 +99,7 @@ def test_remove_duplicates_and_nones_can_remove_duplicates(self): expected = ['testing', 'test of the test', 'another test', 'final test'] # then - the returned data does not have duplicates - self.assertEqual(actual, expected) + self.assertEqual(sorted(actual), sorted(expected)) def test_remove_duplicates_and_nones_can_remove_nones(self): # given - a data processor set up with some missing data of different types (but no duplicates) @@ -116,7 +115,7 @@ def test_remove_duplicates_and_nones_can_remove_nones(self): expected = ['testingtesting', 'another test', 'test of the test', 'final test', 'testing test', 'test of the other test', 'final test but different', 'testing 123', 'a different test', 'testing','final test the final one'] # then - the returned data is the original minus the missing data - self.assertEqual(actual, expected) + self.assertEqual(sorted(actual), sorted(expected)) def test_filter_dates_removes_dates_before_start_date(self): # given - a data processor with start/end date parameters passed in, and data containing dates with some dates outside the start/end range @@ -183,7 +182,7 @@ def test_filter_topics(self): # then - resulting data no longer includes the topic that should be filtered out expected = ['www.test-news/politics/article002', 'www.test-news/tech/article003', 'www.test-news/education/article004','www.test-news/politics/article005', 'www.test-news/tech/article006', 'www.test-news/politics/article007', 'www.test-news/politics/article010', 'www.test-news/tech/article011', 'www.test-news/politics/article012','www.test-news/tech/article013', 'www.test-news/politics/article014', 'www.test-news/tech/article015', 'www.test-news/politics/article016','www.test-news/tech/article017', 'www.test-news/politics/article018', 'www.test-news/politics/article020'] - self.assertEqual(actual, expected) + self.assertEqual(sorted(actual), sorted(expected)) # teardown to undo temp changes after the test suite run - removing temporary test files and directory @classmethod diff --git a/data/tests/test_general_analyser.py b/data/src/tests/test_general_analyser.py similarity index 99% rename from data/tests/test_general_analyser.py rename to data/src/tests/test_general_analyser.py index d244934..4b370a0 100644 --- a/data/tests/test_general_analyser.py +++ b/data/src/tests/test_general_analyser.py @@ -2,9 +2,8 @@ from pathlib import Path import sys sys.path.append(f"{Path(__file__).parent.parent}") -import src -from src.general_analyser import GeneralAnalyser -from src.data_processor import DataProcessor +from general_analyser import GeneralAnalyser +from data_processor import DataProcessor import os import glob from pathlib import Path diff --git a/data/tests/test_multi_source_modeller.py b/data/src/tests/test_multi_source_modeller.py similarity index 99% rename from data/tests/test_multi_source_modeller.py rename to data/src/tests/test_multi_source_modeller.py index b05787b..c4b7c13 100644 --- a/data/tests/test_multi_source_modeller.py +++ b/data/src/tests/test_multi_source_modeller.py @@ -2,9 +2,8 @@ from pathlib import Path import sys sys.path.append(f"{Path(__file__).parent.parent}") -import src -from src.multi_source_modeller import MultiSourceModeller -from src.topic_modeller import TopicModeller +from multi_source_modeller import MultiSourceModeller +from topic_modeller import TopicModeller import os import glob from pathlib import Path diff --git a/data/tests/test_multi_source_sentiments.py b/data/src/tests/test_multi_source_sentiments.py similarity index 99% rename from data/tests/test_multi_source_sentiments.py rename to data/src/tests/test_multi_source_sentiments.py index 1e7357b..4e21ac0 100644 --- a/data/tests/test_multi_source_sentiments.py +++ b/data/src/tests/test_multi_source_sentiments.py @@ -2,8 +2,7 @@ from pathlib import Path import sys sys.path.append(f"{Path(__file__).parent.parent}") -import src -from src.multi_source_sentiments import MultiSourceSentiments +from multi_source_sentiments import MultiSourceSentiments import datetime import os import glob diff --git a/data/tests/test_representative_docs.py b/data/src/tests/test_representative_docs.py similarity index 99% rename from data/tests/test_representative_docs.py rename to data/src/tests/test_representative_docs.py index 30736e8..a571e64 100644 --- a/data/tests/test_representative_docs.py +++ b/data/src/tests/test_representative_docs.py @@ -2,8 +2,7 @@ from pathlib import Path import sys sys.path.append(f"{Path(__file__).parent.parent}") -import src -from src.representative_docs import RepresentativeDocsRepresenter +from representative_docs import RepresentativeDocsRepresenter import os from pathlib import Path import pandas as pd diff --git a/data/tests/test_sentiment.py b/data/src/tests/test_sentiment.py similarity index 99% rename from data/tests/test_sentiment.py rename to data/src/tests/test_sentiment.py index ba6b817..646d3fd 100644 --- a/data/tests/test_sentiment.py +++ b/data/src/tests/test_sentiment.py @@ -2,9 +2,8 @@ from pathlib import Path import sys sys.path.append(f"{Path(__file__).parent.parent}") -import src -from src.sentiment_analyser import SentimentAnalyser -from src.data_processor import DataProcessor +from sentiment_analyser import SentimentAnalyser +from data_processor import DataProcessor import os import glob from pathlib import Path diff --git a/data/tests/test_topic_modeller.py b/data/src/tests/test_topic_modeller.py similarity index 99% rename from data/tests/test_topic_modeller.py rename to data/src/tests/test_topic_modeller.py index eb11afd..d61b908 100644 --- a/data/tests/test_topic_modeller.py +++ b/data/src/tests/test_topic_modeller.py @@ -2,8 +2,7 @@ from pathlib import Path import sys sys.path.append(f"{Path(__file__).parent.parent}") -import src -from src.topic_modeller import TopicModeller +from topic_modeller import TopicModeller import os import glob from pathlib import Path @@ -65,7 +64,7 @@ def test_constructor_adds_data_as_property_of_topic_modeller_which_is_all_lowerc actual_headlines_lowercased = [i.lower() for i in actual_headlines] # then - the topic modeller headline data is all lowercase, sense checking the preprocessing has been applied - self.assertListEqual(actual_headlines, actual_headlines_lowercased) + self.assertListEqual(sorted(actual_headlines), sorted(actual_headlines_lowercased)) def test_model_topics_returns_bertopic_model(self): # given - some data for one source passed into a topic modeller instance, with enough data to get some clusters with the parameters @@ -125,7 +124,7 @@ def test_model_topics_adds_topics_df_with_expected_columns(self): # then - topic info shows expected columns for topic count and name expected_columns = ['Topic', 'Count', 'Name'] - self.assertListEqual(actual_columns, expected_columns) + self.assertListEqual(sorted(actual_columns), sorted(expected_columns)) def test_get_topics_over_time_adds_topics_over_time_property(self): # given - some headline data passed into an instance of the topic modeller class @@ -164,7 +163,7 @@ def test_get_topics_over_time_adds_topics_over_time_df_with_expected_columns(sel # then - the topic modeller's topics_over_time property has the expected columns expected_columns = ['Topic', 'Words', 'Frequency', 'Timestamp'] - self.assertListEqual(actual_columns, expected_columns) + self.assertListEqual(sorted(actual_columns), sorted(expected_columns)) def test_visualise_over_time_returns_plot(self): # given - some headline data passed into an instance of the topic modeller class diff --git a/data/src/topic_modeller.py b/data/src/topic_modeller.py index 13161c3..47a0261 100644 --- a/data/src/topic_modeller.py +++ b/data/src/topic_modeller.py @@ -1,4 +1,4 @@ -from src.data_processor import DataProcessor +from data_processor import DataProcessor import spacy from spacy.lang.en.stop_words import STOP_WORDS from umap import UMAP diff --git a/web-app/public/index.html b/web-app/public/index.html index 4a2592b..51ddc85 100644 --- a/web-app/public/index.html +++ b/web-app/public/index.html @@ -2,7 +2,7 @@ - +