From 3e6a69ce7f109f0fd4352507ad60d58d4cbd24a7 Mon Sep 17 00:00:00 2001 From: Matthias Baumgartner Date: Fri, 25 Nov 2022 14:43:12 +0100 Subject: builders and pipeline --- bsie/tools/builder.py | 217 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 bsie/tools/builder.py (limited to 'bsie/tools/builder.py') diff --git a/bsie/tools/builder.py b/bsie/tools/builder.py new file mode 100644 index 0000000..8f7a410 --- /dev/null +++ b/bsie/tools/builder.py @@ -0,0 +1,217 @@ +""" + +Part of the bsie module. +A copy of the license is provided with the project. +Author: Matthias Baumgartner, 2022 +""" +# imports +import importlib +import logging +import typing + +# bsie imports +from bsie import base +from bsie.base import errors +from bsie.utils.bsfs import URI, typename + +# inner-module imports +from . import pipeline + +# exports +__all__: typing.Sequence[str] = ( + 'ExtractorBuilder', + 'PipelineBuilder', + 'ReaderBuilder', + ) + + +## code ## + +logger = logging.getLogger(__name__) + +def _safe_load(module_name: str, class_name: str): + """Get a class from a module. Raise BuilderError if anything goes wrong.""" + try: + # load the module + module = importlib.import_module(module_name) + except Exception as err: + # cannot import module + raise errors.LoaderError(f'cannot load module {module_name}') from err + + try: + # get the class from the module + cls = getattr(module, class_name) + except Exception as err: + # cannot find the class + raise errors.LoaderError(f'cannot load class {class_name} from module {module_name}') from err + + return cls + + +def _unpack_name(name): + """Split a name into its module and class component (dot-separated).""" + if not isinstance(name, str): + raise TypeError(name) + if '.' not in name: + raise ValueError('name must be a qualified class name.') + module_name, class_name = name[:name.rfind('.')], name[name.rfind('.')+1:] + if module_name == '': + raise ValueError('name must be a qualified class name.') + return module_name, class_name + + +class ReaderBuilder(): + """Build `bsie.base.reader.Reader` instances. + + Readers are defined via their qualified class name + (e.g., bsie.reader.path.Path) and optional keyword + arguments that are passed to the constructor via + the *kwargs* argument (name as key, kwargs as value). + The ReaderBuilder keeps a cache of previously built + reader instances, as they are anyway built with + identical keyword arguments. + + """ + + # keyword arguments + kwargs: typing.Dict[str, typing.Dict[str, typing.Any]] + + # cached readers + cache: typing.Dict[str, base.reader.Reader] + + def __init__(self, kwargs: typing.Dict[str, typing.Dict[str, typing.Any]]): + self.kwargs = kwargs + self.cache = {} + + def build(self, name: str) -> base.reader.Reader: + """Return an instance for the qualified class name.""" + # return cached instance + if name in self.cache: + return self.cache[name] + + # check name and get module/class components + module_name, class_name = _unpack_name(name) + + # import reader class + cls = _safe_load(module_name, class_name) + + # get kwargs + kwargs = self.kwargs.get(name, {}) + if not isinstance(kwargs, dict): + raise TypeError(f'expected a kwargs dict, found {typename(kwargs)}') + + try: # build, cache, and return instance + obj = cls(**kwargs) + # cache instance + self.cache[name] = obj + # return instance + return obj + + except Exception as err: + raise errors.BuilderError(f'failed to build reader {name} due to {typename(err)}: {err}') from err + + +class ExtractorBuilder(): + """Build `bsie.base.extractor.Extractor instances. + + It is permissible to build multiple instances of the same extractor + (typically with different arguments), hence the ExtractorBuilder + receives a list of build specifications. Each specification is + a dict with a single key (extractor's qualified name) and a dict + to be used as keyword arguments. + Example: [{'bsie.extractor.generic.path.Path': {}}, ] + + """ + + # build specifications + specs: typing.List[typing.Dict[str, typing.Dict[str, typing.Any]]] + + def __init__(self, specs: typing.List[typing.Dict[str, typing.Dict[str, typing.Any]]]): + self.specs = specs + + def __iter__(self) -> typing.Iterator[int]: + """Iterate over extractor specifications.""" + return iter(range(len(self.specs))) + + def build(self, index: int) -> base.extractor.Extractor: + """Return an instance of the n'th extractor (n=*index*).""" + # get build instructions + specs = self.specs[index] + + # check specs structure. expecting[{name: {kwargs}}] + if not isinstance(specs, dict): + raise TypeError(f'expected a dict, found {typename(specs)}') + if len(specs) != 1: + raise TypeError(f'expected a dict of length one, found {len(specs)}') + + # get name and args from specs + name = next(iter(specs.keys())) + kwargs = specs[name] + + # check kwargs structure + if not isinstance(kwargs, dict): + raise TypeError(f'expected a dict, found {typename(kwargs)}') + + # check name and get module/class components + module_name, class_name = _unpack_name(name) + + # import extractor class + cls = _safe_load(module_name, class_name) + + try: # build and return instance + return cls(**kwargs) + + except Exception as err: + raise errors.BuilderError(f'failed to build extractor {name} due to {typename(err)}: {err}') from err + + +class PipelineBuilder(): + """Build `bsie.tools.pipeline.Pipeline` instances.""" + + def __init__( + self, + prefix: URI, + reader_builder: ReaderBuilder, + extractor_builder: ExtractorBuilder, + ): + self.prefix = prefix + self.rbuild = reader_builder + self.ebuild = extractor_builder + + def build(self) -> pipeline.Pipeline: + """Return a Pipeline instance.""" + ext2rdr = {} + + for eidx in self.ebuild: + # build extractor + try: + ext = self.ebuild.build(eidx) + + except errors.LoaderError as err: # failed to load extractor; skip + logger.error('failed to load extractor: %s', err) + continue + + except errors.BuilderError as err: # failed to build instance; skip + logger.error(str(err)) + continue + + try: + # get reader required by extractor + if ext.CONTENT_READER is not None: + rdr = self.rbuild.build(ext.CONTENT_READER) + else: + rdr = None + # store extractor + ext2rdr[ext] = rdr + + except errors.LoaderError as err: # failed to load reader + logger.error('failed to load reader: %s', err) + + except errors.BuilderError as err: # failed to build reader + logger.error(str(err)) + + return pipeline.Pipeline(self.prefix, ext2rdr) + + + +## EOF ## -- cgit v1.2.3 From 3b7fee369924eb7704709edeb8c17fff9c020dfb Mon Sep 17 00:00:00 2001 From: Matthias Baumgartner Date: Thu, 15 Dec 2022 17:06:09 +0100 Subject: import fixes --- bsie/tools/builder.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'bsie/tools/builder.py') diff --git a/bsie/tools/builder.py b/bsie/tools/builder.py index 8f7a410..8c6b931 100644 --- a/bsie/tools/builder.py +++ b/bsie/tools/builder.py @@ -13,6 +13,7 @@ import typing from bsie import base from bsie.base import errors from bsie.utils.bsfs import URI, typename +from bsie.utils import bsfs # inner-module imports from . import pipeline @@ -61,7 +62,7 @@ def _unpack_name(name): class ReaderBuilder(): - """Build `bsie.base.reader.Reader` instances. + """Build `bsie.base.Reader` instances. Readers are defined via their qualified class name (e.g., bsie.reader.path.Path) and optional keyword @@ -83,7 +84,7 @@ class ReaderBuilder(): self.kwargs = kwargs self.cache = {} - def build(self, name: str) -> base.reader.Reader: + def build(self, name: str) -> base.Reader: """Return an instance for the qualified class name.""" # return cached instance if name in self.cache: @@ -98,7 +99,7 @@ class ReaderBuilder(): # get kwargs kwargs = self.kwargs.get(name, {}) if not isinstance(kwargs, dict): - raise TypeError(f'expected a kwargs dict, found {typename(kwargs)}') + raise TypeError(f'expected a kwargs dict, found {bsfs.typename(kwargs)}') try: # build, cache, and return instance obj = cls(**kwargs) @@ -108,11 +109,11 @@ class ReaderBuilder(): return obj except Exception as err: - raise errors.BuilderError(f'failed to build reader {name} due to {typename(err)}: {err}') from err + raise errors.BuilderError(f'failed to build reader {name} due to {bsfs.typename(err)}: {err}') from err class ExtractorBuilder(): - """Build `bsie.base.extractor.Extractor instances. + """Build `bsie.base.Extractor instances. It is permissible to build multiple instances of the same extractor (typically with different arguments), hence the ExtractorBuilder @@ -133,14 +134,14 @@ class ExtractorBuilder(): """Iterate over extractor specifications.""" return iter(range(len(self.specs))) - def build(self, index: int) -> base.extractor.Extractor: + def build(self, index: int) -> base.Extractor: """Return an instance of the n'th extractor (n=*index*).""" # get build instructions specs = self.specs[index] # check specs structure. expecting[{name: {kwargs}}] if not isinstance(specs, dict): - raise TypeError(f'expected a dict, found {typename(specs)}') + raise TypeError(f'expected a dict, found {bsfs.typename(specs)}') if len(specs) != 1: raise TypeError(f'expected a dict of length one, found {len(specs)}') @@ -150,7 +151,7 @@ class ExtractorBuilder(): # check kwargs structure if not isinstance(kwargs, dict): - raise TypeError(f'expected a dict, found {typename(kwargs)}') + raise TypeError(f'expected a dict, found {bsfs.typename(kwargs)}') # check name and get module/class components module_name, class_name = _unpack_name(name) -- cgit v1.2.3 From 5d9526783ad8432c7d6dfe18c0e9f2b37950b470 Mon Sep 17 00:00:00 2001 From: Matthias Baumgartner Date: Thu, 15 Dec 2022 17:16:25 +0100 Subject: Pipeline.prefix as Namespace instead of URI --- bsie/tools/builder.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'bsie/tools/builder.py') diff --git a/bsie/tools/builder.py b/bsie/tools/builder.py index 8c6b931..24aea84 100644 --- a/bsie/tools/builder.py +++ b/bsie/tools/builder.py @@ -163,15 +163,24 @@ class ExtractorBuilder(): return cls(**kwargs) except Exception as err: - raise errors.BuilderError(f'failed to build extractor {name} due to {typename(err)}: {err}') from err + raise errors.BuilderError(f'failed to build extractor {name} due to {bsfs.typename(err)}: {err}') from err class PipelineBuilder(): """Build `bsie.tools.pipeline.Pipeline` instances.""" + # Prefix to be used in the Pipeline. + prefix: bsfs.Namespace + + # builder for Readers. + rbuild: ReaderBuilder + + # builder for Extractors. + ebuild: ExtractorBuilder + def __init__( self, - prefix: URI, + prefix: bsfs.Namespace, reader_builder: ReaderBuilder, extractor_builder: ExtractorBuilder, ): -- cgit v1.2.3 From 3b41b2a4b7532c911b63b41066a75b3e1546d214 Mon Sep 17 00:00:00 2001 From: Matthias Baumgartner Date: Thu, 15 Dec 2022 17:21:20 +0100 Subject: minor test improvements and information hiding in builder --- bsie/tools/builder.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) (limited to 'bsie/tools/builder.py') diff --git a/bsie/tools/builder.py b/bsie/tools/builder.py index 24aea84..190d9bf 100644 --- a/bsie/tools/builder.py +++ b/bsie/tools/builder.py @@ -12,7 +12,6 @@ import typing # bsie imports from bsie import base from bsie.base import errors -from bsie.utils.bsfs import URI, typename from bsie.utils import bsfs # inner-module imports @@ -75,20 +74,20 @@ class ReaderBuilder(): """ # keyword arguments - kwargs: typing.Dict[str, typing.Dict[str, typing.Any]] + _kwargs: typing.Dict[str, typing.Dict[str, typing.Any]] # cached readers - cache: typing.Dict[str, base.reader.Reader] + _cache: typing.Dict[str, base.Reader] def __init__(self, kwargs: typing.Dict[str, typing.Dict[str, typing.Any]]): - self.kwargs = kwargs - self.cache = {} + self._kwargs = kwargs + self._cache = {} def build(self, name: str) -> base.Reader: """Return an instance for the qualified class name.""" # return cached instance - if name in self.cache: - return self.cache[name] + if name in self._cache: + return self._cache[name] # check name and get module/class components module_name, class_name = _unpack_name(name) @@ -97,14 +96,14 @@ class ReaderBuilder(): cls = _safe_load(module_name, class_name) # get kwargs - kwargs = self.kwargs.get(name, {}) + kwargs = self._kwargs.get(name, {}) if not isinstance(kwargs, dict): raise TypeError(f'expected a kwargs dict, found {bsfs.typename(kwargs)}') try: # build, cache, and return instance obj = cls(**kwargs) # cache instance - self.cache[name] = obj + self._cache[name] = obj # return instance return obj @@ -125,19 +124,19 @@ class ExtractorBuilder(): """ # build specifications - specs: typing.List[typing.Dict[str, typing.Dict[str, typing.Any]]] + _specs: typing.List[typing.Dict[str, typing.Dict[str, typing.Any]]] def __init__(self, specs: typing.List[typing.Dict[str, typing.Dict[str, typing.Any]]]): - self.specs = specs + self._specs = specs def __iter__(self) -> typing.Iterator[int]: """Iterate over extractor specifications.""" - return iter(range(len(self.specs))) + return iter(range(len(self._specs))) def build(self, index: int) -> base.Extractor: """Return an instance of the n'th extractor (n=*index*).""" # get build instructions - specs = self.specs[index] + specs = self._specs[index] # check specs structure. expecting[{name: {kwargs}}] if not isinstance(specs, dict): -- cgit v1.2.3