1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
|
# standard imports
import os
import unittest
# bsie imports
from bsie.extractor import base
from bsie.matcher import nodes
from bsie.reader.document import Document
from bsie.utils import bsfs, ns
# objects to test
from bsie.extractor.text.metrics import TextMetrics
## code ##
class TestTextMetrics(unittest.TestCase):
def test_schema(self):
self.assertEqual(TextMetrics().schema,
bsfs.schema.from_string(base.SCHEMA_PREAMBLE + '''
bse:num_characters rdfs:subClassOf bsfs:Predicate ;
rdfs:domain bsn:Entity ;
rdfs:range xsd:integer ;
bsfs:unique "true"^^xsd:boolean .
bse:num_paragraphs rdfs:subClassOf bsfs:Predicate ;
rdfs:domain bsn:Entity ;
rdfs:range xsd:integer ;
bsfs:unique "true"^^xsd:boolean .
bse:num_words rdfs:subClassOf bsfs:Predicate ;
rdfs:domain bsn:Entity ;
rdfs:range xsd:integer ;
bsfs:unique "true"^^xsd:boolean .
bse:vocabulary_size rdfs:subClassOf bsfs:Predicate ;
rdfs:domain bsn:Entity ;
rdfs:range xsd:integer ;
bsfs:unique "true"^^xsd:boolean .
bse:vocabulary_entropy rdfs:subClassOf bsfs:Predicate ;
rdfs:domain bsn:Entity ;
rdfs:range xsd:float ;
bsfs:unique "true"^^xsd:boolean .
'''))
def test_extract(self):
# setup
rdr = Document()
ext = TextMetrics()
subject = nodes.Entity(ucid='abc123')
principals = set(ext.principals)
path = os.path.join(os.path.dirname(__file__), 'example-en.txt')
# fetch document
text = rdr(path)
# extracts all specified predicates
triples = set(ext.extract(subject, text, principals))
self.assertSetEqual({(s,p,o) for s,p,o in triples if p.uri != ns.bse.vocabulary_entropy}, {
(subject, ext.schema.predicate(ns.bse.num_characters), 21997),
(subject, ext.schema.predicate(ns.bse.num_paragraphs), 48),
(subject, ext.schema.predicate(ns.bse.num_words), 4234),
(subject, ext.schema.predicate(ns.bse.vocabulary_size), 3510),
})
entropy = {o for s,p,o in triples if p.uri == ns.bse.vocabulary_entropy}
self.assertEqual(len(entropy), 1)
self.assertAlmostEqual(list(entropy)[0], 8.830360505)
# skip unknown predicates
self.assertSetEqual(set(), set(ext.extract(subject, text,
{ext.schema.predicate(ns.bsfs.Predicate).child(ns.bse.unknown)})))
## main ##
if __name__ == '__main__':
unittest.main()
## EOF ##
|
