Newspaper-Topic-Modelling/data/tests/test_representative_docs.py at gh-deploy · CZboop/Newspaper-Topic-Modelling · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
import unittest, pytest
from pathlib import Path
import sys
sys.path.append(f"{Path(__file__).parent.parent}")
import src
from src.representative_docs import RepresentativeDocsRepresenter
import os
from pathlib import Path
import pandas as pd
import shutil
import json

# NOTE: the class being tested simply adds representative docs to an existing plot
# getting representative docs is built in to BERTopic, used in the topic modeller class within this project, and doesn't need to be tested here
# @unittest.skip('skipping for speed while still writing other new tests')
class TestRepresentativeDocs(unittest.TestCase):

    maxDiff = None

    # setup before the test suite run
    @classmethod
    def setUpClass(self):
        # create temp directory for files with test data
        self.test_dir_name = 'temp_test_files'
        self.temp_within_current_dir = f'{Path(__file__).parent}/{self.test_dir_name}'
        Path(self.temp_within_current_dir).mkdir(parents=True, exist_ok=True)

        # also for this class, writing to existing relative path for plots so will create that too
        self.temp_plot_path_parent = f'{Path(__file__).parent}/plots'
        self.temp_plot_path_full = f'{Path(__file__).parent}/plots/plots_with_examples'
        Path(self.temp_plot_path_full).mkdir(parents=True, exist_ok=True)

    # helper function to write files with data to be used in tests
    def setup_write_test_json_file(self, data, filename):
        with open(f'{self.test_dir_name}/{filename}', 'w') as file_:
            json.dump(data, file_)

    def test_reading_in_data_adds_plot_as_class_property(self):
        # given - a fresh instance of the representative docs class and json files with a simplified topic plotly plot and a list of representative docs for the topics on that plot
        test_data_plot = {"data" : [{"customdata" : [[0, "cluster | title | but | fake", 100], [1, "a | different | test | topic", 100], [2, "cluster | title | once | again", 100], [3, "the | words | of | cluster", 100], [4, "topic | definition | but | testing", 100], [5, "a | topic | in | test", 100], [6, "name | of | the | group", 100], [7, "a | bag | of | docs", 100], [8, "chunks | of | data | here", 100], [9, "some | words | in | group", 100]],"hovertemplate":"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}"}]}
        self.setup_write_test_json_file(test_data_plot, 'test_topics.json')

        test_data_docs = {"2" : ["example 1 group 2", "example 2 group 2"], "4" : ["example 1 group 4", "example 2 group 4", "example 3 group 4"], "6" : ["example 1 group 6"], "8" : ["example 1 group 8", "example 2 group 8"], "1" : ["example 1 group 1", "example 2 group 1"], "3" : ["example 1 group 3", "example 1 group 3"], "5" : ["example 1 group 5"], "7" : ["example 1 group 7", "example 2 group 7"], "9" : ["example 1 group 9", "example 2 group 9", "example 3 group 9"]}
        self.setup_write_test_json_file(test_data_docs, 'test.json')

        undertest_class = RepresentativeDocsRepresenter(path_to_plot = 'temp_test_files', path_to_repr_docs = 'temp_test_files', sources=['test'])

        # when - the read data method is called for the given source
        undertest_class._read_data(source='test')

        # then - the json plot of the topic clusters is added as a class property to the undertest class
        self.assertTrue(hasattr(undertest_class, "plot"))

    def test_reading_in_data_adds_representative_docs_as_class_property(self):
        # given - a fresh instance of the representative docs class and json files with a simplified topic plotly plot and a list of representative docs for the topics on that plot
        test_data_plot = {"data" : [{"customdata" : [[0, "cluster | title | but | fake", 100], [1, "a | different | test | topic", 100], [2, "cluster | title | once | again", 100], [3, "the | words | of | cluster", 100], [4, "topic | definition | but | testing", 100], [5, "a | topic | in | test", 100], [6, "name | of | the | group", 100], [7, "a | bag | of | docs", 100], [8, "chunks | of | data | here", 100], [9, "some | words | in | group", 100]],"hovertemplate":"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}"}]}
        self.setup_write_test_json_file(test_data_plot, 'test_topics.json')

        test_data_docs = {"2" : ["example 1 group 2", "example 2 group 2"], "4" : ["example 1 group 4", "example 2 group 4", "example 3 group 4"], "6" : ["example 1 group 6"], "8" : ["example 1 group 8", "example 2 group 8"], "1" : ["example 1 group 1", "example 2 group 1"], "3" : ["example 1 group 3", "example 1 group 3"], "5" : ["example 1 group 5"], "7" : ["example 1 group 7", "example 2 group 7"], "9" : ["example 1 group 9", "example 2 group 9", "example 3 group 9"]}
        self.setup_write_test_json_file(test_data_docs, 'test.json')

        undertest_class = RepresentativeDocsRepresenter(path_to_plot = 'temp_test_files', path_to_repr_docs = 'temp_test_files', sources=['test'])

        # when - the read data method is called for the given source
        undertest_class._read_data(source='test')

        # then - the json data of the representative docs is added as a class property to the undertest class
        self.assertTrue(hasattr(undertest_class, "repr_docs"))

    def test_adding_representative_docs_to_plot(self):
        # given - a fresh instance of the representative docs class and json files with a simplified topic plotly plot and a list of representative docs for the topics on that plot, where data has been read in
        test_data_plot = {"data" : [{"customdata" : [[0, "cluster | title | but | fake", 100], [1, "a | different | test | topic", 100], [2, "cluster | title | once | again", 100], [3, "the | words | of | cluster", 100], [4, "topic | definition | but | testing", 100], [5, "a | topic | in | test", 100], [6, "name | of | the | group", 100], [7, "a | bag | of | docs", 100], [8, "chunks | of | data | here", 100], [9, "some | words | in | group", 100]],"hovertemplate":"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}"}]}
        self.setup_write_test_json_file(test_data_plot, 'test_topics.json')

        test_data_docs = {"0" : ["example 1 group 0", "example 2 group 0"], "2" : ["example 1 group 2", "example 2 group 2"], "4" : ["example 1 group 4", "example 2 group 4", "example 3 group 4"], "6" : ["example 1 group 6"], "8" : ["example 1 group 8", "example 2 group 8"], "1" : ["example 1 group 1", "example 2 group 1"], "3" : ["example 1 group 3", "example 1 group 3"], "5" : ["example 1 group 5"], "7" : ["example 1 group 7", "example 2 group 7"], "9" : ["example 1 group 9", "example 2 group 9", "example 3 group 9"]}
        self.setup_write_test_json_file(test_data_docs, 'test.json')

        undertest_class = RepresentativeDocsRepresenter(path_to_plot = 'temp_test_files', path_to_repr_docs = 'temp_test_files', sources=['test'])
        undertest_class._read_data(source="test")

        # when - the add representative docs method is called for the given source
        actual = undertest_class.add_repr_docs(source="test")

        # then - a new json plot is returned with the first representative doc added based on topic number not order
        expected = {"data" : [{"customdata" : [[0, "cluster | title | but | fake", 100, "example 1 group 0"], [1, "a | different | test | topic", 100, "example 1 group 1"], [2, "cluster | title | once | again", 100, "example 1 group 2"], [3, "the | words | of | cluster", 100, "example 1 group 3"], [4, "topic | definition | but | testing", 100, "example 1 group 4"], [5, "a | topic | in | test", 100, "example 1 group 5"], [6, "name | of | the | group", 100, "example 1 group 6"], [7, "a | bag | of | docs", 100, "example 1 group 7"], [8, "chunks | of | data | here", 100, "example 1 group 8"], [9, "some | words | in | group", 100, "example 1 group 9"]],"hovertemplate":"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}<br>Example: %{customdata[3]}"}]}

        self.assertEqual(actual, expected)

    def test_running_for_all_sources(self):
        # given - an instance of the undertest class with multiple sources
        for i in range(3):
            test_data_plot = {"data" : [{"customdata" : [[0, "cluster | title | but | fake", 100], [1, "a | different | test | topic", 100], [2, "cluster | title | once | again", 100], [3, "the | words | of | cluster", 100], [4, "topic | definition | but | testing", 100], [5, "a | topic | in | test", 100], [6, "name | of | the | group", 100], [7, "a | bag | of | docs", 100], [8, "chunks | of | data | here", 100], [9, "some | words | in | group", 100]],"hovertemplate":"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}"}]}
            self.setup_write_test_json_file(test_data_plot, f'test{i}_topics.json')

            test_data_docs = {"0" : ["example 1 group 0", "example 2 group 0"], "2" : ["example 1 group 2", "example 2 group 2"], "4" : ["example 1 group 4", "example 2 group 4", "example 3 group 4"], "6" : ["example 1 group 6"], "8" : ["example 1 group 8", "example 2 group 8"], "1" : ["example 1 group 1", "example 2 group 1"], "3" : ["example 1 group 3", "example 1 group 3"], "5" : ["example 1 group 5"], "7" : ["example 1 group 7", "example 2 group 7"], "9" : ["example 1 group 9", "example 2 group 9", "example 3 group 9"]}
            self.setup_write_test_json_file(test_data_docs, f'test{i}.json')

        undertest_class = RepresentativeDocsRepresenter(path_to_plot = 'temp_test_files', path_to_repr_docs = 'temp_test_files', sources=['test0', 'test1', 'test2'])

        # when - the run for all sources method is called
        actual = undertest_class.run_for_all_sources()

        # then - as many updated plots as sources are returned
        expected = [{"data" : [{"customdata" : [[0, "cluster | title | but | fake", 100, "example 1 group 0"], [1, "a | different | test | topic", 100, "example 1 group 1"], [2, "cluster | title | once | again", 100, "example 1 group 2"], [3, "the | words | of | cluster", 100, "example 1 group 3"], [4, "topic | definition | but | testing", 100, "example 1 group 4"], [5, "a | topic | in | test", 100, "example 1 group 5"], [6, "name | of | the | group", 100, "example 1 group 6"], [7, "a | bag | of | docs", 100, "example 1 group 7"], [8, "chunks | of | data | here", 100, "example 1 group 8"], [9, "some | words | in | group", 100, "example 1 group 9"]],"hovertemplate":"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}<br>Example: %{customdata[3]}"}]}, {"data" : [{"customdata" : [[0, "cluster | title | but | fake", 100, "example 1 group 0"], [1, "a | different | test | topic", 100, "example 1 group 1"], [2, "cluster | title | once | again", 100, "example 1 group 2"], [3, "the | words | of | cluster", 100, "example 1 group 3"], [4, "topic | definition | but | testing", 100, "example 1 group 4"], [5, "a | topic | in | test", 100, "example 1 group 5"], [6, "name | of | the | group", 100, "example 1 group 6"], [7, "a | bag | of | docs", 100, "example 1 group 7"], [8, "chunks | of | data | here", 100, "example 1 group 8"], [9, "some | words | in | group", 100, "example 1 group 9"]],"hovertemplate":"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}<br>Example: %{customdata[3]}"}]}, {"data" : [{"customdata" : [[0, "cluster | title | but | fake", 100, "example 1 group 0"], [1, "a | different | test | topic", 100, "example 1 group 1"], [2, "cluster | title | once | again", 100, "example 1 group 2"], [3, "the | words | of | cluster", 100, "example 1 group 3"], [4, "topic | definition | but | testing", 100, "example 1 group 4"], [5, "a | topic | in | test", 100, "example 1 group 5"], [6, "name | of | the | group", 100, "example 1 group 6"], [7, "a | bag | of | docs", 100, "example 1 group 7"], [8, "chunks | of | data | here", 100, "example 1 group 8"], [9, "some | words | in | group", 100, "example 1 group 9"]],"hovertemplate":"<b>Topic %{customdata[0]}</b><br>Words: %{customdata[1]}<br>Size: %{customdata[2]}<br>Example: %{customdata[3]}"}]}]

        self.assertEqual(actual, expected)

    @classmethod
    def tearDownClass(self):
        try:
            shutil.rmtree(self.temp_within_current_dir)
        except OSError as error:
            print(f'An error occured while trying to delete directory: {error.filename} - {error.strerror}.')
        try:
            shutil.rmtree(self.temp_plot_path_parent)
        except OSError as error:
            print(f'An error occured while trying to delete directory: {error.filename} - {error.strerror}.')

if __name__ == "__main__":
    unittest.main()