From e94368c75468e3e94382b12705e55d396249eaca Mon Sep 17 00:00:00 2001 From: Matthias Baumgartner Date: Sun, 18 Dec 2022 14:20:25 +0100 Subject: bsfs applications --- .gitignore | 1 + bsfs.app | 52 +++++++++++++++++++++++++++ bsfs/apps/__init__.py | 20 +++++++++++ bsfs/apps/init.py | 73 +++++++++++++++++++++++++++++++++++++ bsfs/apps/migrate.py | 67 ++++++++++++++++++++++++++++++++++ bsfs/utils/errors.py | 3 ++ test/apps/__init__.py | 0 test/apps/config.json | 8 +++++ test/apps/schema-1.nt | 19 ++++++++++ test/apps/schema-2.nt | 19 ++++++++++ test/apps/test_init.py | 91 +++++++++++++++++++++++++++++++++++++++++++++++ test/apps/test_migrate.py | 66 ++++++++++++++++++++++++++++++++++ 12 files changed, 419 insertions(+) create mode 100755 bsfs.app create mode 100644 bsfs/apps/__init__.py create mode 100644 bsfs/apps/init.py create mode 100644 bsfs/apps/migrate.py create mode 100644 test/apps/__init__.py create mode 100644 test/apps/config.json create mode 100644 test/apps/schema-1.nt create mode 100644 test/apps/schema-2.nt create mode 100644 test/apps/test_init.py create mode 100644 test/apps/test_migrate.py diff --git a/.gitignore b/.gitignore index de722e6..ba88570 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ __pycache__ bsfs.egg-info htmlcov tags +dev/ env # dist builds diff --git a/bsfs.app b/bsfs.app new file mode 100755 index 0000000..babacbb --- /dev/null +++ b/bsfs.app @@ -0,0 +1,52 @@ +"""BSFS tools. + +Part of the BlackStar filesystem (bsfs) module. +A copy of the license is provided with the project. +Author: Matthias Baumgartner, 2022 +""" +# imports +import argparse +import typing + +# module imports +import bsfs +import bsfs.apps + +# exports +__all__: typing.Sequence[str] = ( + 'main', + ) + +# config +apps = { + 'init' : bsfs.apps.init, + 'migrate' : bsfs.apps.migrate, + } + + +## code ## + +def main(argv): + """Black Star File System maintenance tools.""" + parser = argparse.ArgumentParser(description=main.__doc__, prog='bsfs') + # version + parser.add_argument('--version', action='version', + version='%(prog)s version {}.{}.{}'.format(*bsfs.version_info)) + # application selection + parser.add_argument('app', choices=apps.keys(), + help='Select the application to run.') + # dangling args + parser.add_argument('rest', nargs=argparse.REMAINDER) + # parse + args = parser.parse_args() + # run application + apps[args.app](args.rest) + + +## main ## + +if __name__ == '__main__': + import sys + main(sys.argv[1:]) + +## EOF ## diff --git a/bsfs/apps/__init__.py b/bsfs/apps/__init__.py new file mode 100644 index 0000000..7efaa87 --- /dev/null +++ b/bsfs/apps/__init__.py @@ -0,0 +1,20 @@ +""" + +Part of the BlackStar filesystem (bsfs) module. +A copy of the license is provided with the project. +Author: Matthias Baumgartner, 2022 +""" +# imports +import typing + +# inner-module imports +from .init import main as init +from .migrate import main as migrate + +# exports +__all__: typing.Sequence[str] = ( + 'init', + 'migrate', + ) + +## EOF ## diff --git a/bsfs/apps/init.py b/bsfs/apps/init.py new file mode 100644 index 0000000..3e2ef37 --- /dev/null +++ b/bsfs/apps/init.py @@ -0,0 +1,73 @@ +""" + +Part of the BlackStar filesystem (bsfs) module. +A copy of the license is provided with the project. +Author: Matthias Baumgartner, 2022 +""" +# imports +import argparse +import json +import sys +import typing + +# bsfs imports +from bsfs.utils import errors + +# exports +__all__: typing.Sequence[str] = ( + 'main', + ) + +## code ## + +def init_sparql_store(user) -> typing.Any: + """Initialize a SparqlStore backend. Returns a configuration to load it.""" + # nothing to do for non-persistent store + # return config to storage + return { + 'Graph': { + 'user': user, + 'backend': { + 'SparqlStore': {}, + }, + } + } + + +def main(argv): + """Create a new bsfs storage structure.""" + parser = argparse.ArgumentParser(description=main.__doc__, prog='init') + # global arguments + parser.add_argument('--user', type=str, default='http://example.com/me', + help='Default user.') + parser.add_argument('--output', type=str, default=None, + help='Write the config to a file instead of standard output.') + #parser.add_argument('--schema', type=str, default=None, + # help='Initial schema.') + # storage selection + parser.add_argument('store', choices=('sparql', ), + help='Which storage to initialize.') + # storage args + # parse args + args = parser.parse_args(argv) + + # initialize selected storage + if args.store == 'sparql': + config = init_sparql_store(args.user) + else: + raise errors.UnreachableError() + + # print config + if args.output is not None: + with open(args.output, mode='wt', encoding='UTF-8') as ofile: + json.dump(config, ofile) + else: + json.dump(config, sys.stdout) + + +## main ## + +if __name__ == '__main__': + main(sys.argv[1:]) + +## EOF ## diff --git a/bsfs/apps/migrate.py b/bsfs/apps/migrate.py new file mode 100644 index 0000000..91c1661 --- /dev/null +++ b/bsfs/apps/migrate.py @@ -0,0 +1,67 @@ +""" + +Part of the BlackStar filesystem (bsfs) module. +A copy of the license is provided with the project. +Author: Matthias Baumgartner, 2022 +""" +# imports +import argparse +import json +import logging +import sys +import typing + +# bsfs imports +import bsfs + +# exports +__all__: typing.Sequence[str] = ( + 'main', + ) + + +## code ## + +logger = logging.getLogger(__name__) + +def main(argv): + """Migrate a storage structure to a modified schema.""" + parser = argparse.ArgumentParser(description=main.__doc__, prog='migrate') + parser.add_argument('--remove', action='store_true', default=False, + help='Remove classes that are not specified in the provided schema.') + parser.add_argument('config', type=str, default=None, + help='Path to the storage config file.') + parser.add_argument('schema', nargs=argparse.REMAINDER, + help='Paths to schema files. Reads from standard input if no file is supplied.') + args = parser.parse_args(argv) + + # load storage config + with open(args.config, mode='rt', encoding='UTF-8') as ifile: + config = json.load(ifile) + # open bsfs storage + graph = bsfs.Open(config) + + # initialize schema + schema = bsfs.schema.Schema.Empty() + if len(args.schema) == 0: + # assemble schema from standard input + schema = schema + bsfs.schema.Schema.from_string(sys.stdin.read()) + else: + # assemble schema from input files + for pth in args.schema: + with open(pth, mode='rt', encoding='UTF-8') as ifile: + schema = schema + bsfs.schema.Schema.from_string(ifile.read()) + + # migrate schema + graph.migrate(schema, not args.remove) + + # return the migrated storage + return graph + + +## main ## + +if __name__ == '__main__': + main(sys.argv[1:]) + +## EOF ## diff --git a/bsfs/utils/errors.py b/bsfs/utils/errors.py index 04561a2..c5e8e16 100644 --- a/bsfs/utils/errors.py +++ b/bsfs/utils/errors.py @@ -35,4 +35,7 @@ class ProgrammingError(_BSFSError): class UnreachableError(ProgrammingError): """Bravo, you've reached a point in code that should logically not be reachable.""" +class ConfigError(_BSFSError): + """User config issue.""" + ## EOF ## diff --git a/test/apps/__init__.py b/test/apps/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/apps/config.json b/test/apps/config.json new file mode 100644 index 0000000..ffc5ef7 --- /dev/null +++ b/test/apps/config.json @@ -0,0 +1,8 @@ +{ + "Graph": { + "user": "http://example.com/me", + "backend": { + "SparqlStore": {} + } + } +} diff --git a/test/apps/schema-1.nt b/test/apps/schema-1.nt new file mode 100644 index 0000000..e57146d --- /dev/null +++ b/test/apps/schema-1.nt @@ -0,0 +1,19 @@ + +prefix rdfs: +prefix xsd: + +# common bsfs prefixes +prefix bsfs: +prefix bse: + +# essential nodes +bsfs:Entity rdfs:subClassOf bsfs:Node . + +# common definitions +xsd:string rdfs:subClassOf bsfs:Literal . + +bse:filename rdfs:subClassOf bsfs:Predicate ; + rdfs:domain bsfs:Entity ; + rdfs:range xsd:string ; + bsfs:unique "false"^^xsd:boolean . + diff --git a/test/apps/schema-2.nt b/test/apps/schema-2.nt new file mode 100644 index 0000000..525ac99 --- /dev/null +++ b/test/apps/schema-2.nt @@ -0,0 +1,19 @@ + +prefix rdfs: +prefix xsd: + +# common bsfs prefixes +prefix bsfs: +prefix bse: + +# essential nodes +bsfs:Entity rdfs:subClassOf bsfs:Node . + +# common definitions +xsd:integer rdfs:subClassOf bsfs:Literal . + +bse:filesize rdfs:subClassOf bsfs:Predicate ; + rdfs:domain bsfs:Entity ; + rdfs:range xsd:integer ; + bsfs:unique "true"^^xsd:boolean . + diff --git a/test/apps/test_init.py b/test/apps/test_init.py new file mode 100644 index 0000000..bae6a68 --- /dev/null +++ b/test/apps/test_init.py @@ -0,0 +1,91 @@ +""" + +Part of the bsfs test suite. +A copy of the license is provided with the project. +Author: Matthias Baumgartner, 2022 +""" +# imports +import contextlib +import io +import json +import os +import tempfile +import unittest + +# bsie imports +from bsfs.front import build_graph +from bsfs.graph import Graph + +# objects to test +from bsfs.apps.init import main, init_sparql_store + + +## code ## + +class TestInit(unittest.TestCase): + def test_main(self): + + # cannot pass an invalid store + with contextlib.redirect_stderr(io.StringIO()): + self.assertRaises(SystemExit, main, ['--user', 'http://example.com/me', 'foobar']) + + # produces a config structure + outbuf = io.StringIO() + with contextlib.redirect_stdout(outbuf): + main(['--user', 'http://example.com/me', 'sparql']) + self.assertEqual(json.loads(outbuf.getvalue()), { + 'Graph': { + 'user': 'http://example.com/me', + 'backend': { + 'SparqlStore': {}}}}) + # config is valid + self.assertIsInstance(build_graph(json.loads(outbuf.getvalue())), Graph) + + # respects user flag + outbuf = io.StringIO() + with contextlib.redirect_stdout(outbuf): + main(['--user', 'http://example.com/you', 'sparql']) + self.assertEqual(json.loads(outbuf.getvalue()), { + 'Graph': { + 'user': 'http://example.com/you', + 'backend': { + 'SparqlStore': {}}}}) + + # respects output flag + _, path = tempfile.mkstemp(prefix='bsfs-test-', text=True) + outbuf = io.StringIO() + with contextlib.redirect_stdout(outbuf): + main(['--user', 'http://example.com/me', '--output', path, 'sparql']) + with open(path, 'rt') as ifile: + config = ifile.read() + os.unlink(path) + self.assertEqual(outbuf.getvalue(), '') + self.assertEqual(json.loads(config), { + 'Graph': { + 'user': 'http://example.com/me', + 'backend': { + 'SparqlStore': {}}}}) + + def test_init_sparql_store(self): + # returns a config structure + self.assertEqual(init_sparql_store('http://example.com/me'), { + 'Graph': { + 'user': 'http://example.com/me', + 'backend': { + 'SparqlStore': {}}}}) + # respects user + self.assertEqual(init_sparql_store('http://example.com/you'), { + 'Graph': { + 'user': 'http://example.com/you', + 'backend': { + 'SparqlStore': {}}}}) + # the config is valid + self.assertIsInstance(build_graph(init_sparql_store('http://example.com/me')), Graph) + + +## main ## + +if __name__ == '__main__': + unittest.main() + +## EOF ## diff --git a/test/apps/test_migrate.py b/test/apps/test_migrate.py new file mode 100644 index 0000000..957509a --- /dev/null +++ b/test/apps/test_migrate.py @@ -0,0 +1,66 @@ +""" + +Part of the bsfs test suite. +A copy of the license is provided with the project. +Author: Matthias Baumgartner, 2022 +""" +# imports +import contextlib +import io +import os +import sys +import unittest +import unittest.mock + +# bsie imports +from bsfs.schema import Schema + +# objects to test +from bsfs.apps.migrate import main + + +## code ## + +class TestMigrate(unittest.TestCase): + def test_main(self): + config = os.path.join(os.path.dirname(__file__), 'config.json') + schema_1 = os.path.join(os.path.dirname(__file__), 'schema-1.nt') + schema_2 = os.path.join(os.path.dirname(__file__), 'schema-2.nt') + + # provide no config + with contextlib.redirect_stderr(io.StringIO()): + self.assertRaises(SystemExit, main, []) + + # read schema from file + with open(schema_1) as ifile: + target = Schema.from_string(ifile.read()) + graph = main([config, schema_1]) + self.assertTrue(target <= graph.schema) + + # read schema from multiple files + with open(schema_1) as ifile: + target = Schema.from_string(ifile.read()) + with open(schema_2) as ifile: + target = target + Schema.from_string(ifile.read()) + graph = main([config, schema_1, schema_2]) + self.assertTrue(target <= graph.schema) + + # read schema from stdin + with open(schema_1, 'rt') as ifile: + target = Schema.from_string(ifile.read()) + with open(schema_1, 'rt') as ifile: + with unittest.mock.patch('sys.stdin', ifile): + graph = main([config]) + self.assertTrue(target <= graph.schema) + + # remove predicates + # NOTE: cannot currently test this since there's nothing to remove in the loaded (empty) schema. + + +## main ## + +if __name__ == '__main__': + unittest.main() + +## EOF ## + -- cgit v1.2.3