Newspaper-Topic-Modelling/data/src/tests/test_representative_docs.py at main · CZboop/Newspaper-Topic-Modelling · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
import unittest, pytest
from pathlib import Path
import sys
sys.path.append(f"{Path(__file__).parent.parent}")
from representative_docs import RepresentativeDocsRepresenter
import os
from pathlib import Path
import pandas as pd
import shutil
import json

# NOTE: the class being tested simply adds representative docs to an existing plot
# getting representative docs is built in to BERTopic, used in the topic modeller class within this project, and doesn't need to be tested here
# @unittest.skip('skipping for speed while still writing other new tests')
class TestRepresentativeDocs(unittest.TestCase):

    maxDiff = None

    # setup before the test suite run
    @classmethod
    def setUpClass(self):
        # create temp directory for files with test data
        self.test_dir_name = 'temp_test_files'
        self.temp_within_current_dir = f'{Path(__file__).parent}/{self.test_dir_name}'
        Path(self.temp_within_current_dir).mkdir(parents=True, exist_ok=True)

        # also for this class, writing to existing relative path for plots so will create that too
        self.temp_plot_path_parent = f'{Path(__file__).parent}/plots'
        self.temp_plot_path_full = f'{Path(__file__).parent}/plots/plots_with_examples'
        Path(self.temp_plot_path_full).mkdir(parents=True, exist_ok=True)

    # helper function to write files with data to be used in tests
    def setup_write_test_json_file(self, data, filename):
        with open(f'{self.test_dir_name}/{filename}', 'w') as file_:
            json.dump(data, file_)

    def test_reading_in_data_adds_plot_as_class_property(self):
        # given - a fresh instance of the representative docs class and json files with a simplified topic plotly plot and a list of representative docs for the topics on that plot
        test_data_plot = {"data" : [{"customdata" : [[0, "cluster | title | but | fake", 100], [1, "a | different | test | topic", 100], [2, "cluster | title | once | again", 100], [3, "the | words | of | cluster", 100], [4, "topic | definition | but | testing", 100], [5, "a | topic | in | test", 100], [6, "name | of | the | group", 100], [7, "a | bag | of | docs", 100], [8, "chunks | of | data | here", 100], [9, "some | words | in | group", 100]],"hovertemplate":"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}"}]}
        self.setup_write_test_json_file(test_data_plot, 'test_topics.json')

        test_data_docs = {"2" : ["example 1 group 2", "example 2 group 2"], "4" : ["example 1 group 4", "example 2 group 4", "example 3 group 4"], "6" : ["example 1 group 6"], "8" : ["example 1 group 8", "example 2 group 8"], "1" : ["example 1 group 1", "example 2 group 1"], "3" : ["example 1 group 3", "example 1 group 3"], "5" : ["example 1 group 5"], "7" : ["example 1 group 7", "example 2 group 7"], "9" : ["example 1 group 9", "example 2 group 9", "example 3 group 9"]}
        self.setup_write_test_json_file(test_data_docs, 'test.json')

        undertest_class = RepresentativeDocsRepresenter(path_to_plot = 'temp_test_files', path_to_repr_docs = 'temp_test_files', sources=['test'])

        # when - the read data method is called for the given source
        undertest_class._read_data(source='test')

        # then - the json plot of the topic clusters is added as a class property to the undertest class
        self.assertTrue(hasattr(undertest_class, "plot"))

    def test_reading_in_data_adds_representative_docs_as_class_property(self):
        # given - a fresh instance of the representative docs class and json files with a simplified topic plotly plot and a list of representative docs for the topics on that plot
        test_data_plot = {"data" : [{"customdata" : [[0, "cluster | title | but | fake", 100], [1, "a | different | test | topic", 100], [2, "cluster | title | once | again", 100], [3, "the | words | of | cluster", 100], [4, "topic | definition | but | testing", 100], [5, "a | topic | in | test", 100], [6, "name | of | the | group", 100], [7, "a | bag | of | docs", 100], [8, "chunks | of | data | here", 100], [9, "some | words | in | group", 100]],"hovertemplate":"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}"}]}
        self.setup_write_test_json_file(test_data_plot, 'test_topics.json')

        test_data_docs = {"2" : ["example 1 group 2", "example 2 group 2"], "4" : ["example 1 group 4", "example 2 group 4", "example 3 group 4"], "6" : ["example 1 group 6"], "8" : ["example 1 group 8", "example 2 group 8"], "1" : ["example 1 group 1", "example 2 group 1"], "3" : ["example 1 group 3", "example 1 group 3"], "5" : ["example 1 group 5"], "7" : ["example 1 group 7", "example 2 group 7"], "9" : ["example 1 group 9", "example 2 group 9", "example 3 group 9"]}
        self.setup_write_test_json_file(test_data_docs, 'test.json')

        undertest_class = RepresentativeDocsRepresenter(path_to_plot = 'temp_test_files', path_to_repr_docs = 'temp_test_files', sources=['test'])

        # when - the read data method is called for the given source
        undertest_class._read_data(source='test')

        # then - the json data of the representative docs is added as a class property to the undertest class
        self.assertTrue(hasattr(undertest_class, "repr_docs"))

    def test_adding_representative_docs_to_plot(self):
        # given - a fresh instance of the representative docs class and json files with a simplified topic plotly plot and a list of representative docs for the topics on that plot, where data has been read in
        test_data_plot = {"data" : [{"customdata" : [[0, "cluster | title | but | fake", 100], [1, "a | different | test | topic", 100], [2, "cluster | title | once | again", 100], [3, "the | words | of | cluster", 100], [4, "topic | definition | but | testing", 100], [5, "a | topic | in | test", 100], [6, "name | of | the | group", 100], [7, "a | bag | of | docs", 100], [8, "chunks | of | data | here", 100], [9, "some | words | in | group", 100]],"hovertemplate":"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}"}]}
        self.setup_write_test_json_file(test_data_plot, 'test_topics.json')

        test_data_docs = {"0" : ["example 1 group 0", "example 2 group 0"], "2" : ["example 1 group 2", "example 2 group 2"], "4" : ["example 1 group 4", "example 2 group 4", "example 3 group 4"], "6" : ["example 1 group 6"], "8" : ["example 1 group 8", "example 2 group 8"], "1" : ["example 1 group 1", "example 2 group 1"], "3" : ["example 1 group 3", "example 1 group 3"], "5" : ["example 1 group 5"], "7" : ["example 1 group 7", "example 2 group 7"], "9" : ["example 1 group 9", "example 2 group 9", "example 3 group 9"]}
        self.setup_write_test_json_file(test_data_docs, 'test.json')

        undertest_class = RepresentativeDocsRepresenter(path_to_plot = 'temp_test_files', path_to_repr_docs = 'temp_test_files', sources=['test'])
        undertest_class._read_data(source="test")

        # when - the add representative docs method is called for the given source
        actual = undertest_class.add_repr_docs(source="test")

        # then - a new json plot is returned with the first representative doc added based on topic number not order
        expected = {"data" : [{"customdata" : [[0, "cluster | title | but | fake", 100, "example 1 group 0"], [1, "a | different | test | topic", 100, "example 1 group 1"], [2, "cluster | title | once | again", 100, "example 1 group 2"], [3, "the | words | of | cluster", 100, "example 1 group 3"], [4, "topic | definition | but | testing", 100, "example 1 group 4"], [5, "a | topic | in | test", 100, "example 1 group 5"], [6, "name | of | the | group", 100, "example 1 group 6"], [7, "a | bag | of | docs", 100, "example 1 group 7"], [8, "chunks | of | data | here", 100, "example 1 group 8"], [9, "some | words | in | group", 100, "example 1 group 9"]],"hovertemplate":"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}<br>Example: %{customdata[3]}"}]}

        self.assertEqual(actual, expected)

    def test_running_for_all_sources(self):
        # given - an instance of the undertest class with multiple sources
        for i in range(3):
            test_data_plot = {"data" : [{"customdata" : [[0, "cluster | title | but | fake", 100], [1, "a | different | test | topic", 100], [2, "cluster | title | once | again", 100], [3, "the | words | of | cluster", 100], [4, "topic | definition | but | testing", 100], [5, "a | topic | in | test", 100], [6, "name | of | the | group", 100], [7, "a | bag | of | docs", 100], [8, "chunks | of | data | here", 100], [9, "some | words | in | group", 100]],"hovertemplate":"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}"}]}
            self.setup_write_test_json_file(test_data_plot, f'test{i}_topics.json')

            test_data_docs = {"0" : ["example 1 group 0", "example 2 group 0"], "2" : ["example 1 group 2", "example 2 group 2"], "4" : ["example 1 group 4", "example 2 group 4", "example 3 group 4"], "6" : ["example 1 group 6"], "8" : ["example 1 group 8", "example 2 group 8"], "1" : ["example 1 group 1", "example 2 group 1"], "3" : ["example 1 group 3", "example 1 group 3"], "5" : ["example 1 group 5"], "7" : ["example 1 group 7", "example 2 group 7"], "9" : ["example 1 group 9", "example 2 group 9", "example 3 group 9"]}
            self.setup_write_test_json_file(test_data_docs, f'test{i}.json')

        undertest_class = RepresentativeDocsRepresenter(path_to_plot = 'temp_test_files', path_to_repr_docs = 'temp_test_files', sources=['test0', 'test1', 'test2'])

        # when - the run for all sources method is called
        actual = undertest_class.run_for_all_sources()

        # then - as many updated plots as sources are returned
        expected = [{"data" : [{"customdata" : [[0, "cluster | title | but | fake", 100, "example 1 group 0"], [1, "a | different | test | topic", 100, "example 1 group 1"], [2, "cluster | title | once | again", 100, "example 1 group 2"], [3, "the | words | of | cluster", 100, "example 1 group 3"], [4, "topic | definition | but | testing", 100, "example 1 group 4"], [5, "a | topic | in | test", 100, "example 1 group 5"], [6, "name | of | the | group", 100, "example 1 group 6"], [7, "a | bag | of | docs", 100, "example 1 group 7"], [8, "chunks | of | data | here", 100, "example 1 group 8"], [9, "some | words | in | group", 100, "example 1 group 9"]],"hovertemplate":"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}<br>Example: %{customdata[3]}"}]}, {"data" : [{"customdata" : [[0, "cluster | title | but | fake", 100, "example 1 group 0"], [1, "a | different | test | topic", 100, "example 1 group 1"], [2, "cluster | title | once | again", 100, "example 1 group 2"], [3, "the | words | of | cluster", 100, "example 1 group 3"], [4, "topic | definition | but | testing", 100, "example 1 group 4"], [5, "a | topic | in | test", 100, "example 1 group 5"], [6, "name | of | the | group", 100, "example 1 group 6"], [7, "a | bag | of | docs", 100, "example 1 group 7"], [8, "chunks | of | data | here", 100, "example 1 group 8"], [9, "some | words | in | group", 100, "example 1 group 9"]],"hovertemplate":"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}<br>Example: %{customdata[3]}"}]}, {"data" : [{"customdata" : [[0, "cluster | title | but | fake", 100, "example 1 group 0"], [1, "a | different | test | topic", 100, "example 1 group 1"], [2, "cluster | title | once | again", 100, "example 1 group 2"], [3, "the | words | of | cluster", 100, "example 1 group 3"], [4, "topic | definition | but | testing", 100, "example 1 group 4"], [5, "a | topic | in | test", 100, "example 1 group 5"], [6, "name | of | the | group", 100, "example 1 group 6"], [7, "a | bag | of | docs", 100, "example 1 group 7"], [8, "chunks | of | data | here", 100, "example 1 group 8"], [9, "some | words | in | group", 100, "example 1 group 9"]],"hovertemplate":"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}<br>Example: %{customdata[3]}"}]}]

        self.assertEqual(actual, expected)

    @classmethod
    def tearDownClass(self):
        try:
            shutil.rmtree(self.temp_within_current_dir)
        except OSError as error:
            print(f'An error occured while trying to delete directory: {error.filename} - {error.strerror}.')
        try:
            shutil.rmtree(self.temp_plot_path_parent)
        except OSError as error:
            print(f'An error occured while trying to delete directory: {error.filename} - {error.strerror}.')

if __name__ == "__main__":
    unittest.main()