From 9c366758665d9cfee7796ee45a8167a5412ae9ae Mon Sep 17 00:00:00 2001 From: Matthias Baumgartner Date: Fri, 13 Jan 2023 09:49:10 +0100 Subject: filter early port, parsing adaptions --- .gitignore | 8 +- tagit/actions/__init__.py | 14 +- tagit/actions/filter.py | 33 +- tagit/apps/port-config.yaml | 4 + tagit/assets/icons/scalable/filter/add.svg | 102 ++++ tagit/assets/icons/scalable/filter/address.svg | 124 ++++ tagit/assets/icons/scalable/filter/go_back.svg | 158 +++++ tagit/assets/icons/scalable/filter/go_forth.svg | 154 +++++ tagit/assets/icons/scalable/filter/shingles.svg | 217 +++++++ tagit/dialogues/__init__.py | 14 +- tagit/dialogues/autoinput.py | 73 +++ tagit/dialogues/dialogue.kv | 114 ++++ tagit/dialogues/dialogue.py | 108 ++++ tagit/dialogues/error.py | 45 ++ tagit/dialogues/license.t | 33 ++ tagit/dialogues/simple_input.kv | 30 + tagit/dialogues/simple_input.py | 55 ++ tagit/dialogues/stoken.py | 40 ++ tagit/parsing/__init__.py | 8 +- tagit/parsing/filter.py | 374 ++++++++++++ tagit/parsing/search.py | 405 ------------- tagit/parsing/sort.py | 17 +- tagit/utils/__init__.py | 1 + tagit/utils/bsfs.py | 10 +- tagit/utils/namespaces.py | 30 + tagit/utils/shared.py | 12 + tagit/widgets/filter.py | 13 +- tagit/widgets/session.py | 4 + test/parsing/test_filter.py | 751 ++++++++++++++++++++++++ test/parsing/test_search.py | 707 ---------------------- 30 files changed, 2490 insertions(+), 1168 deletions(-) create mode 100644 tagit/assets/icons/scalable/filter/add.svg create mode 100644 tagit/assets/icons/scalable/filter/address.svg create mode 100644 tagit/assets/icons/scalable/filter/go_back.svg create mode 100644 tagit/assets/icons/scalable/filter/go_forth.svg create mode 100644 tagit/assets/icons/scalable/filter/shingles.svg create mode 100644 tagit/dialogues/autoinput.py create mode 100644 tagit/dialogues/dialogue.kv create mode 100644 tagit/dialogues/dialogue.py create mode 100644 tagit/dialogues/error.py create mode 100644 tagit/dialogues/license.t create mode 100644 tagit/dialogues/simple_input.kv create mode 100644 tagit/dialogues/simple_input.py create mode 100644 tagit/dialogues/stoken.py create mode 100644 tagit/parsing/filter.py delete mode 100644 tagit/parsing/search.py create mode 100644 tagit/utils/namespaces.py create mode 100644 test/parsing/test_filter.py delete mode 100644 test/parsing/test_search.py diff --git a/.gitignore b/.gitignore index f9c6fcf..767d1af 100644 --- a/.gitignore +++ b/.gitignore @@ -32,9 +32,9 @@ tagit/external/setproperty/setproperty.c tagit/external/setproperty/setproperty.cpython* # assets -tagit/assets/icons/kivy/browser/ -tagit/assets/icons/kivy/misc/ -tagit/assets/icons/kivy/planes/ - +tagit/assets/icons/kivy/browser* +tagit/assets/icons/kivy/filter* +tagit/assets/icons/kivy/misc* +tagit/assets/icons/kivy/planes* ## EOF ## diff --git a/tagit/actions/__init__.py b/tagit/actions/__init__.py index 034c4a1..c34cbe8 100644 --- a/tagit/actions/__init__.py +++ b/tagit/actions/__init__.py @@ -57,13 +57,13 @@ class ActionBuilder(BuilderBase): 'SelectInvert': browser.SelectInvert, 'Select': browser.Select, ## filter - #'AddToken': filter.AddToken, - #'RemoveToken': filter.RemoveToken, - #'SetToken': filter.SetToken, - #'EditToken': filter.EditToken, - #'GoBack': filter.GoBack, - #'GoForth': filter.GoForth, - #'JumpToToken': filter.JumpToToken, + 'AddToken': filter.AddToken, + 'RemoveToken': filter.RemoveToken, + 'SetToken': filter.SetToken, + 'EditToken': filter.EditToken, + 'GoBack': filter.GoBack, + 'GoForth': filter.GoForth, + 'JumpToToken': filter.JumpToToken, #'SearchByAddressOnce': filter.SearchByAddressOnce, #'SearchmodeSwitch': filter.SearchmodeSwitch, ## grouping diff --git a/tagit/actions/filter.py b/tagit/actions/filter.py index 3702879..e878952 100644 --- a/tagit/actions/filter.py +++ b/tagit/actions/filter.py @@ -12,14 +12,12 @@ from kivy.lang import Builder import kivy.properties as kp # tagit imports -from tagit import config -from tagit import dialogues -#from tagit.parsing import ParserError # FIXME: mb/port -#from tagit.parsing.search import ast_from_string, ast_to_string, ast # FIXME: mb/port -#from tagit.storage.base import ns # FIXME: mb/port -from tagit.utils import Frame -from tagit.widgets.bindings import Binding +from tagit import config, dialogues +from tagit.utils import errors, Frame +from tagit.widgets import Binding from tagit.widgets.filter import FilterAwareMixin +#from tagit.parsing.search import ast_to_string, ast # FIXME: mb/port +#from tagit.storage.base import ns # FIXME: mb/port # inner-module imports from .action import Action @@ -54,7 +52,7 @@ class SetToken(Action): with self.root.filter as filter: try: # parse filter into tokens - tokens = list(ast_from_string(text)) + tokens = list(self.root.session.filter_from_string(text)) # grab current frame filter.f_head.append(self.root.browser.frame) @@ -93,7 +91,10 @@ class AddToken(Action): def apply(self, token=None): if token is None: - sugg = {node.label for node in self.root.session.storage.all(ns.tagit.storage.base.Tag)} + #sugg = {node.label for node in self.root.session.storage.all(ns.tagit.storage.base.Tag)} + # FIXME: mb/port/bsfs + #sugg = set(self.root.session.storage.all(ns.bsfs.Tag).label()) + sugg = {'hello', 'world'} dlg = dialogues.TokenEdit(suggestions=sugg) dlg.bind(on_ok=lambda wx: self.add_from_string(wx.text)) dlg.open() @@ -104,8 +105,8 @@ class AddToken(Action): def add_from_string(self, text): try: - self.add_token(ast_from_string(text)) - except ParserError as e: + self.add_token(self.root.session.filter_from_string(text)) + except errors.ParserError as e: dialogues.Error(text=f'syntax error: {e}').open() def add_token(self, tokens): @@ -128,8 +129,10 @@ class EditToken(Action): text = kp.StringProperty('Edit token') def apply(self, token): - sugg = {node.label for node in self.root.session.storage.all(ns.tagit.storage.base.Tag)} - text = ast_to_string(token) + #sugg = {node.label for node in self.root.session.storage.all(ns.tagit.storage.base.Tag)} # FIXME: mb/port + sugg = {'hello', 'world'} # FIXME: mb/port + #text = ast_to_string(token) + text = 'hello world' dlg = dialogues.TokenEdit(text=text, suggestions=sugg) dlg.bind(on_ok=lambda obj: self.on_ok(token, obj)) dlg.open() @@ -137,8 +140,8 @@ class EditToken(Action): def on_ok(self, token, obj): with self.root.filter as filter: try: - tokens_from_text = ast_from_string(obj.text) - except ParserError as e: + tokens_from_text = self.root.session.filter_from_string(obj.text) # FIXME: mb/port + except errors.ParserError as e: dialogues.Error(text=f'Invalid token: {e}').open() return diff --git a/tagit/apps/port-config.yaml b/tagit/apps/port-config.yaml index 501eacd..d5c45e9 100644 --- a/tagit/apps/port-config.yaml +++ b/tagit/apps/port-config.yaml @@ -20,6 +20,7 @@ ui: - ShowDashboard #- AddTag #- EditTag + - AddToken #- CreateGroup #- DissolveGroup #- SelectAll @@ -30,6 +31,9 @@ ui: #- SelectSingle #- SelectMulti #- SelectRange + filter: + - AddToken + - EditToken context: app: - ShowSettings diff --git a/tagit/assets/icons/scalable/filter/add.svg b/tagit/assets/icons/scalable/filter/add.svg new file mode 100644 index 0000000..a544053 --- /dev/null +++ b/tagit/assets/icons/scalable/filter/add.svg @@ -0,0 +1,102 @@ + + + + + + + + + + + + + image/svg+xml + + + + + + + + + diff --git a/tagit/assets/icons/scalable/filter/address.svg b/tagit/assets/icons/scalable/filter/address.svg new file mode 100644 index 0000000..de30faa --- /dev/null +++ b/tagit/assets/icons/scalable/filter/address.svg @@ -0,0 +1,124 @@ + + + + + + + + + + + + + image/svg+xml + + + + + + + + + foo + + diff --git a/tagit/assets/icons/scalable/filter/go_back.svg b/tagit/assets/icons/scalable/filter/go_back.svg new file mode 100644 index 0000000..a972c87 --- /dev/null +++ b/tagit/assets/icons/scalable/filter/go_back.svg @@ -0,0 +1,158 @@ + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + diff --git a/tagit/assets/icons/scalable/filter/go_forth.svg b/tagit/assets/icons/scalable/filter/go_forth.svg new file mode 100644 index 0000000..f4a246d --- /dev/null +++ b/tagit/assets/icons/scalable/filter/go_forth.svg @@ -0,0 +1,154 @@ + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + diff --git a/tagit/assets/icons/scalable/filter/shingles.svg b/tagit/assets/icons/scalable/filter/shingles.svg new file mode 100644 index 0000000..a07b6ea --- /dev/null +++ b/tagit/assets/icons/scalable/filter/shingles.svg @@ -0,0 +1,217 @@ + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tagit/dialogues/__init__.py b/tagit/dialogues/__init__.py index bee5bf4..d7792cb 100644 --- a/tagit/dialogues/__init__.py +++ b/tagit/dialogues/__init__.py @@ -18,11 +18,11 @@ import typing # inner-module imports ##from .spash import Splash -#from .autoinput import AutoTextInput +from .autoinput import AutoTextInput #from .console import Console #from .dir_creator import DirCreator #from .dir_picker import DirPicker -#from .error import Error +from .error import Error #from .export import Export #from .file_creator import FileCreator #from .file_picker import FilePicker @@ -32,8 +32,8 @@ import typing #from .path_picker import PathPicker #from .progress import Progress #from .project import Project -#from .simple_input import SimpleInput -#from .stoken import TokenEdit +from .simple_input import SimpleInput +from .stoken import TokenEdit #from .yesno import YesNo # exports @@ -41,7 +41,7 @@ __all__: typing.Sequence[str] = ( #'Console', #'DirCreator', #'DirPicker', - #'Error', + 'Error', #'Export', #'FileCreator', #'FilePicker', @@ -51,8 +51,8 @@ __all__: typing.Sequence[str] = ( #'PathPicker', #'Progress', #'Project', - #'SimpleInput', - #'TokenEdit', + 'SimpleInput', + 'TokenEdit', #'YesNo', ) diff --git a/tagit/dialogues/autoinput.py b/tagit/dialogues/autoinput.py new file mode 100644 index 0000000..a036ed4 --- /dev/null +++ b/tagit/dialogues/autoinput.py @@ -0,0 +1,73 @@ +"""This is a simple example of how to use suggestion text. + +In this example you setup a word_list at the begining. In this case +'the the quick brown fox jumps over the lazy old dog'. This list along +with any new word written word in the textinput is available as a +suggestion when you are typing. You can press tab to auto complete the text. + +Based on & thanks to akshayaurora: + https://gist.github.com/akshayaurora/fa5a68980af585e355668e5adce5f98b + +Part of the tagit module. +A copy of the license is provided with the project. +Modifications authored by: Matthias Baumgartner, 2022 +""" +# standard imports +from bisect import bisect + +# kivy imports +from kivy.uix.textinput import TextInput +import kivy.properties as kp + +# exports +__all__ = ('AutoTextInput', ) + + +## code ## + +class AutoTextInput(TextInput): + + sep = kp.StringProperty(',') + suffix = kp.StringProperty(' ') + vocabulary = kp.ListProperty() + + def on_suggestion_text(self, wx, value): + if not value: + return + + super(AutoTextInput, self).on_suggestion_text(wx, value) + + def keyboard_on_key_down(self, window, keycode, text, modifiers): + if self.suggestion_text and keycode[1] == 'tab': # complete suggestion_text + self.insert_text(self.suggestion_text + self.sep + self.suffix) + self.suggestion_text = '' + return True + return super(AutoTextInput, self).keyboard_on_key_down(window, keycode, text, modifiers) + + def on_text(self, wx, value): + # include all current text from textinput into the word list + # the kind of behavior sublime text has + + # what's on the current line + temp = value[:value.rfind(self.sep)].split(self.sep) + temp = [s.strip() for s in temp] + # combine with static vocabulary + wordlist = sorted(set(self.vocabulary + temp)) + + # get prefix + prefix = value[value.rfind(self.sep)+1:].strip() + if not prefix: + return + + # binary search on (sorted) wordlist + pos = bisect(wordlist, prefix) + + # check if matching string found + if pos == len(wordlist) or not wordlist[pos].startswith(prefix): + self.suggestion_text = '' + return + + # fetch suffix from wordlist + self.suggestion_text = wordlist[pos][len(prefix):] + +## EOF ## diff --git a/tagit/dialogues/dialogue.kv b/tagit/dialogues/dialogue.kv new file mode 100644 index 0000000..e23f0db --- /dev/null +++ b/tagit/dialogues/dialogue.kv @@ -0,0 +1,114 @@ +#:import get_root tagit.utils.get_root +# FIXME: remove need for get_root + +<-Dialogue>: + auto_dismiss: True + ok_on_enter: True + +: + + orientation: 'vertical' + padding: '12dp' + size_hint: 0.66, None + height: self.minimum_height + + canvas: + # mask main window + Color: + rgba: 0,0,0, 0.7 * self.parent._anim_alpha + Rectangle: + size: self.parent._window.size if self.parent._window else (0, 0) + + # solid background color + Color: + rgb: 1, 1, 1 + BorderImage: + source: self.parent.background + border: self.parent.border + pos: self.pos + size: self.size + + +: + # nothing to do + +: + title: '' + title_color: 1,1,1,1 + + Label: + text: root.title + size_hint_y: None + height: self.texture_size[1] + dp(16) + text_size: self.width - dp(16), None + font_size: '16sp' + color: root.title_color + bold: True + halign: 'center' + valing: 'middle' + + canvas.before: + # Background + Color: + rgb: 0.2, 0.2, 0.2 + Rectangle: + size: self.size + pos: self.pos + + # top border + #Color: + # rgb: 0.5, 0.5, 0.5 + #Line: + # points: self.x, self.y + self.height, self.x + self.width, self.y + self.height + # width: 2 + + # bottom border + #Color: + # rgb: 0.5, 0.5, 0.5 + #Line: + # points: self.x, self.y, self.x + self.width, self.y + # width: 2 + + # small space + Label: + size_hint_y: None + height: 12 + + +: + orientation: 'vertical' + size_hint_y: None + height: dp(48+8) + + # small space + Label: + size_hint_y: None + height: dp(8) + + # here come the buttons + + +: + ok_text: 'OK' + + Button: + text: root.ok_text + on_press: get_root(self).ok() + + +: + cancel_text: 'Cancel' + ok_text: 'OK' + ok_enabled: True + + BoxLayout: + orientation: 'horizontal' + Button: + text: root.cancel_text + on_press: get_root(self).cancel() + Button: + text: root.ok_text + on_press: get_root(self).ok() + disabled: not root.ok_enabled + +## EOF ## diff --git a/tagit/dialogues/dialogue.py b/tagit/dialogues/dialogue.py new file mode 100644 index 0000000..1aa0e9a --- /dev/null +++ b/tagit/dialogues/dialogue.py @@ -0,0 +1,108 @@ +"""Popup dialogue. + +Rougly based on code from https://gist.github.com/kived/742397a80d61e6be225a +by Ryan Pessa. The license is provided in the source folder. + +Part of the tagit module. +A copy of the license is provided with the project. +Modifications authored by: Matthias Baumgartner, 2022 +""" +# standard imports +import os + +# kivy imports +from kivy.lang import Builder +from kivy.uix.boxlayout import BoxLayout +from kivy.uix.popup import Popup +import kivy.properties as kp + +# exports +__all__ = ('Dialogue', ) + + +## code ## + +# Load kv +Builder.load_file(os.path.join(os.path.dirname(__file__), 'dialogue.kv')) + +# classes +class Dialogue(Popup): + """Popup dialogue base class. + + Use like below: + + >>> dlg = Dialogue() + >>> dlg.bind(on_ok=....) + >>> dlg.open() + + """ + + ok_on_enter = kp.BooleanProperty() + + __events__ = ('on_ok', 'on_cancel') + + def __init__(self, *args, **kwargs): + super(Dialogue, self).__init__(*args, **kwargs) + from kivy.core.window import Window + # assumes that the first widget created controls the keyboard + #Window.children[-1].request_exclusive_keyboard() + # Alternatively, you can bind a function (self._on_keyboard) to on_keyboard + # which returns True. This stops the event from being processed by the main + # window. + # However, this still does not keep the 'enter' from 'on_text_validate' from + # being processed by the main window. + Window.bind(on_keyboard=self._on_keyboard) + # By binding to on_key_down, the key can trigger the ok action. + # This also prevents the enter event to be processed by the main window, + # unlike the 'on_text_validate' of TextInput. + Window.bind(on_key_down=self._key_down) + + def _on_keyboard(self, *args, **kwargs): + # block events from processing in the main window + return True + + def _key_down(self, instance, key, scancode, codepoint, modifiers): + if key == 13 and self.ok_on_enter: + self.ok() + return True + # must not stop other events such that ctrl up/down reach the browser + + def ok(self): + """User pressed the OK button.""" + self.dispatch('on_ok') + self.dismiss() + + def cancel(self): + """User pressed the Cancel button.""" + self.dispatch('on_cancel') + self.dismiss() + + def on_dismiss(self): + from kivy.core.window import Window + # assumes that the first widget created controls the keyboard + #Window.children[-1].release_exclusive_keyboard() + Window.unbind(on_keyboard=self._on_keyboard) + Window.unbind(on_key_down=self._key_down) + super(Dialogue, self).on_dismiss() + + def on_ok(self): + """Event prototype.""" + pass + + def on_cancel(self): + """Event prototype.""" + pass + +# helper classes + +# content bases +class DialogueContentBase(BoxLayout): pass +class DialogueContentTitle(DialogueContentBase): pass +class DialogueContentNoTitle(DialogueContentBase): pass + +# buttons +class DialogueButtons(BoxLayout): pass +class DialogueButtons_One(DialogueButtons): pass +class DialogueButtons_Two(DialogueButtons): pass + +## EOF ## diff --git a/tagit/dialogues/error.py b/tagit/dialogues/error.py new file mode 100644 index 0000000..d93f853 --- /dev/null +++ b/tagit/dialogues/error.py @@ -0,0 +1,45 @@ +"""Dialogue to show an error message. + +Part of the tagit module. +A copy of the license is provided with the project. +Author: Matthias Baumgartner, 2022 +""" +# kivy imports +from kivy.lang import Builder +import kivy.properties as kp + +# inner-module imports +from .dialogue import Dialogue + +# exports +__all__ = ('Error', ) + + +## code ## + +# load kv +Builder.load_string(''' +: + text: '' + ok_on_enter: False + + DialogueContentNoTitle: + + Label: + markup: True + text: root.text + size_hint_y: None + color: 1, 0, 0, 1 + height: self.texture_size[1] + dp(16) + text_size: self.width - dp(16), None + halign: 'center' + + DialogueButtons_One: +''') + +# classes +class Error(Dialogue): + """Error message.""" + text = kp.StringProperty('') + +## EOF ## diff --git a/tagit/dialogues/license.t b/tagit/dialogues/license.t new file mode 100644 index 0000000..bbd2830 --- /dev/null +++ b/tagit/dialogues/license.t @@ -0,0 +1,33 @@ + +The dialogues are based on the following code: + +https://gist.github.com/kived/742397a80d61e6be225a + +It ships with license, provided below: + +>>> The following license shall apply to all Public Gists owned by account. It +>>> shall never apply to any Secret Gists, for which no license of any sort is +>>> granted. +>>> +>>> Copyright (c) 2015- Ryan Pessa +>>> +>>> Permission is hereby granted, free of charge, to any person obtaining a copy +>>> of this software and associated documentation files (the "Software"), to deal +>>> in the Software without restriction, including without limitation the rights +>>> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +>>> copies of the Software, and to permit persons to whom the Software is +>>> furnished to do so, subject to the following conditions: +>>> +>>> The above copyright notice and this permission notice shall be included in +>>> all copies or substantial portions of the Software. +>>> +>>> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +>>> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +>>> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +>>> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +>>> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +>>> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +>>> THE SOFTWARE. + +Code modification are subject to the license of the tagit software. + diff --git a/tagit/dialogues/simple_input.kv b/tagit/dialogues/simple_input.kv new file mode 100644 index 0000000..b7deb9c --- /dev/null +++ b/tagit/dialogues/simple_input.kv @@ -0,0 +1,30 @@ + +#:import AutoTextInput tagit.dialogues + +: + text: '' + ok_on_enter: True + cancel_on_defocus: True + + DialogueContentNoTitle: + + #AutoTextInput: + TextInput: + vocabulary: root.suggestions + sep: root.suggestion_sep + suffix: root.suggestion_suffix + focus: True + text: root.text + size_hint_y: None + multiline: False + height: self.minimum_height + text_size: self.width - dp(16), None + halign: 'center' + + on_text: root.text = self.text + on_focus: root.on_text_focus(*args) + #on_text_validate: root.ok() # handled via the ok_on_enter mechanism + + DialogueButtons_Two: + +## EOF ## diff --git a/tagit/dialogues/simple_input.py b/tagit/dialogues/simple_input.py new file mode 100644 index 0000000..d7cc69f --- /dev/null +++ b/tagit/dialogues/simple_input.py @@ -0,0 +1,55 @@ +"""Dialogue with a single-line text input field. + +Part of the tagit module. +A copy of the license is provided with the project. +Author: Matthias Baumgartner, 2022 +""" +# standard imports +import os + +# kivy imports +from kivy.lang import Builder +import kivy.properties as kp + +# inner-module imports +from .dialogue import Dialogue + +# exports +__all__ = ('SimpleInput', ) + + +## code ## + +# load kv +Builder.load_file(os.path.join(os.path.dirname(__file__), 'simple_input.kv')) + +# classes +class SimpleInput(Dialogue): + """Dialogue with a single-line text input field. + + Pass the default text as **text**. + + >>> SimpleInput(text='Hello world').open() + + In case of touch events, they need to be inhibited to change the focus. + + >>> FocusBehavior.ignored_touch.append(touch) + + """ + + # Defocus problem: + # Buttons defocus when on_press, but on_release is ok. + # Touch events must be blocked via FocusBehavior + + text = kp.StringProperty('') + cancel_on_defocus = kp.BooleanProperty(True) + suggestions = kp.ListProperty() + suggestion_sep = kp.StringProperty(',') + suggestion_suffix = kp.StringProperty(' ') + + + def on_text_focus(self, instance, focus): + if not focus and self.cancel_on_defocus: + self.dismiss() + +## EOF ## diff --git a/tagit/dialogues/stoken.py b/tagit/dialogues/stoken.py new file mode 100644 index 0000000..6e5427a --- /dev/null +++ b/tagit/dialogues/stoken.py @@ -0,0 +1,40 @@ +"""Search token editor + +Part of the tagit module. +A copy of the license is provided with the project. +Author: Matthias Baumgartner, 2022 + +""" +# kivy imports +from kivy.lang import Builder +import kivy.properties as kp + +# inner-module imports +from .simple_input import SimpleInput + +# exports +__all__ = ('TokenEdit', ) + + +## code ## + +# Load kv +Builder.load_string(''' +#:import AutoTextInput tagit.dialogues + +: + text: '' + ok_on_enter: True + cancel_on_defocus: True +''') + +# classes +class TokenEdit(SimpleInput): + """Search token editor + """ + # TODO: Currently this is no different than SimpleInput. + # It should be extend to specify the type and getting help + # with editing ranges and alternative selection. + pass + +## EOF ## diff --git a/tagit/parsing/__init__.py b/tagit/parsing/__init__.py index 1c431a4..0070bf9 100644 --- a/tagit/parsing/__init__.py +++ b/tagit/parsing/__init__.py @@ -6,14 +6,14 @@ Author: Matthias Baumgartner, 2022 """ # inner-module imports from .datefmt import parse_datetime -from .search import ast_from_string -from .sort import sort_from_string +from .filter import Filter +from .sort import Sort # exports __all__ = ( - 'ast_from_string', + 'Filter', + 'Sort', 'parse_datetime', - 'sort_from_string', ) ## EOF ## diff --git a/tagit/parsing/filter.py b/tagit/parsing/filter.py new file mode 100644 index 0000000..ea8df51 --- /dev/null +++ b/tagit/parsing/filter.py @@ -0,0 +1,374 @@ +"""User-specified search query parsing. + +>>> q = "has mime / tag in (november, october) / ! Apfel / time < 10.10.2004 / iso in (100, 200)" +>>> ast = filter_from_string(q) + +Part of the tagit module. +A copy of the license is provided with the project. +Author: Matthias Baumgartner, 2022 +""" +# standard imports +from datetime import datetime + +# external imports +from pyparsing import CaselessKeyword, Combine, Group, Optional, Or, Word, delimitedList, nums, oneOf, ParseException, Literal, QuotedString, alphanums, alphas8bit, punc8bit + +# tagit imports +from tagit.utils import bsfs, errors, ns, ttime +from tagit.utils.bsfs import ast + +# inner-module imports +from .datefmt import parse_datetime + +# constants +SEARCH_DELIM = '/' +VALUE_DELIM = ',' + +# exports +__all__ = ( + 'Filter', + ) + + +## code ## + +class Filter(): + + # parsers + _DATETIME_PREDICATES = None + _QUERY = None + + def __init__(self, schema: bsfs.schema.Schema): + self.schema = schema + + @property + def schema(self) -> bsfs.schema.Schema: + return self._schema + + @schema.setter + def schema(self, schema: bsfs.schema.Schema): + self._schema = schema + self.build_parser() + + def build_parser(self): + """ + """ + # valid predicates per type, as supplied by tagit.library + # FIXME: + # * range / type constraints + # * how to filter predicates + # * distinguish between prefix and label + """ + Categorical: string, float, integer; labelled node (tag, group); maybe boolean + Continuous: float, integer + Datetime: datetime + Existencial: all of the above, particularly bool; unllabelled node (preview) + -> rfds:range + + > Target: Entity (allow others?) -> rfds:domain + > Require: searchable as specified in backend AND user-searchable as specified in frontend + """ + # all relevant predicates + predicates = {pred for pred in self.schema.predicates() if pred.domain <= self.schema.node(ns.bsfs.Entity)} + # filter through accept/reject lists + ... # FIXME + # shortcuts + self._abb2uri = {pred.uri.fragment: pred.uri for pred in predicates} # FIXME: tie-breaking for duplicates + self._uri2abb = {uri: fragment for fragment, uri in self._abb2uri.items()} + # all predicates + _PREDICATES = {self._uri2abb[pred.uri] for pred in predicates} + # numeric predicates + _PREDICATES_NUMERIC = {self._uri2abb[pred.uri] for pred in predicates if isinstance(pred.range, bsfs.schema.Literal) and pred.range <= self.schema.literal(ns.bsfs.Number)} # FIXME: type check might become unnecessary + # datetime predicates + self._DATETIME_PREDICATES = {pred.uri for pred in predicates if isinstance(pred.range, bsfs.schema.Literal) and pred.range <= self.schema.literal(ns.bsfs.Time)} # FIXME: type check might become unnecessary + _PREDICATES_DATETIME = {self._uri2abb[pred] for pred in self._DATETIME_PREDICATES} + + + # terminal symbols + number = Group(Optional(oneOf('- +')) \ + + Combine(Word(nums) + Optional('.' + Optional(Word(nums))))) + words = QuotedString(quoteChar='"', escChar='\\') \ + ^ QuotedString(quoteChar="'", escChar='\\') \ + ^ Word(alphanums + alphas8bit + punc8bit + ' *#%&-.:;<=>?@^_`{}~') + # FIXME: allow escaped chars "( ) / , [ ]" + # FIXME: Non-ascii characters + + # predicates + predicate = Or([CaselessKeyword(p) for p in _PREDICATES]).setResultsName( + 'predicate') + date_predicate = Or([CaselessKeyword(p) for p in _PREDICATES_DATETIME]).setResultsName( + 'predicate') + num_predicate = Or([CaselessKeyword(p) for p in _PREDICATES_NUMERIC]).setResultsName( + 'predicate') + + # existence + """ + EXPR := has PREDICATE | has no PREDICATE + PREDICATE := [predicate] + """ + op = (CaselessKeyword('has') ^ CaselessKeyword('has no') ^ CaselessKeyword('has not')).setResultsName('op') + _EXISTENCE = Group(op + predicate).setResultsName('existence') + + + # continuous + """ + EXPR := PREDICATE CMP VALUE | VALUE CMP PREDICATE CMP VALUE | PREDICATE OP RANGE + PREDICATE := [predicate] + CMP := < | <= | = | >= | > + OP := : | = | in | not in + RANGE := BOPEN VALUE RSEP VALUE BCLOSE | BOPEN RSEP VALUE BLOSE | BOPEN VALUE RSEP BCLOSE + BOPEN := ( | [ | ] + BCLOSE := ) | ] | [ + RSEP := : | - + VALUE := [digit] | [date] + """ + # range style + rsepn = oneOf(': -') + rsepd = Literal('-') + bclose = oneOf(') ] [').setResultsName('bclose') + bopen = oneOf('( [ ]').setResultsName('bopen') + op = Or([':', '=', 'in']).setResultsName('op') + datefmt = parse_datetime.DATETIME + rngn = num_predicate + op + bopen + number('lo') + rsepn + number('hi') + bclose ^ \ + num_predicate + op + bopen + rsepn + number('hi') + bclose ^ \ + num_predicate + op + bopen + number('lo') + rsepn + bclose + rngd = date_predicate + op + bopen + datefmt('lo') + rsepd + datefmt('hi') + bclose ^ \ + date_predicate + op + bopen + rsepd + datefmt('hi') + bclose ^ \ + date_predicate + op + bopen + datefmt('lo') + rsepd + bclose + # equation style + cmp = oneOf('< <= = >= >').setResultsName('cmp') + eqn = num_predicate + cmp('cright') + number('vright') ^ \ + number('vleft') + cmp('cleft') + num_predicate ^ \ + number('vleft') + cmp('cleft') + num_predicate + cmp('cright') + number('vright') + eqd = date_predicate + cmp('cright') + datefmt('vright') ^ \ + datefmt('vleft') + cmp('cleft') + date_predicate ^ \ + datefmt('vleft') + cmp('cleft') + date_predicate + cmp('cright') + datefmt('vright') + # combined + _CONTINUOUS = Group( + Group(eqn).setResultsName('eq') ^ + Group(eqd).setResultsName('eq') ^ + Group(rngn).setResultsName('range') ^ \ + Group(rngd).setResultsName('range') \ + ).setResultsName('continuous') + + + # categorical + """ + EXPR := PREDICATE OP VALUE | PREDICATE OP (VALUE) + PREDICATE := [predicate] + OP := : | = | in | not in | != | ~ | !~ + VALUE := TERM | VALUE, + TERM := [word] + """ + op = (CaselessKeyword('in') ^ CaselessKeyword('not in') ^ ':' ^ '=' ^ '!=' ^ '~' ^ '!~').setResultsName('op') + value = delimitedList(words, delim=VALUE_DELIM).setResultsName('value') + _CATEGORICAL = Group(predicate + op + ('(' + value + ')' | value) ).setResultsName('categorical') + + + # tag shortcuts + """ + EXPR := OP VALUE | OP (VALUE) | VALUE | (VALUE) + OP := ! | ~ | !~ + VALUE := TERM | VALUE, + TERM := [word] + """ + op = oneOf('! ~ !~').setResultsName('op') + value = delimitedList(words, delim=VALUE_DELIM).setResultsName('value') + _TAG = Group(Optional(op) + '(' + value + ')' ^ Optional(op) + value).setResultsName('tag') + + + # overall query + """ + QUERY := QUERY / QUERY | EXPR + """ + self._QUERY = delimitedList(_EXISTENCE | _CONTINUOUS | _CATEGORICAL | _TAG, delim=SEARCH_DELIM) + return self + + def __call__(self, search): + try: + parsed = self._QUERY.parseString(search, parseAll=True) + except ParseException as e: + raise errors.ParserError('Cannot parse query', e) + + # convert to AST + tokens = [] + for exp in parsed: + if exp.getName() == 'existence': + pred = self._abb2uri[exp.predicate.lower()] + if 'op' not in exp: # prevented by grammar + raise errors.ParserError('Missing operator', exp) + elif exp.op == 'has': + tok = ast.filter.Has(pred) + elif exp.op in ('has no', 'has not'): + tok = ast.filter.Not(ast.filter.Has(pred)) + else: # prevented by grammar + raise errors.ParserError('Invalid operator ({})'.format(exp.op), exp) + tokens.append(tok) + + elif exp.getName() == 'categorical': + pred = self._abb2uri[exp.predicate.lower()] + approx = False + values = [s.strip() for s in exp.value] + if 'op' not in exp: # prevented by grammar + raise errors.ParserError('Missing operator', exp) + if exp.op in ('~' '!~'): + approx = True + if exp.op in (':', '=', '~', 'in'): + tok = ast.filter.Any(pred, ast.filter.Includes(*values, approx=approx)) + elif exp.op in ('!=', '!~', 'not in'): + tok = ast.filter.All(pred, ast.filter.Excludes(*values, approx=approx)) + else: # prevented by grammar + raise errors.ParserError('Invalid operator ({})'.format(exp.op), exp) + tokens.append(tok) + + elif exp.getName() == 'tag': + values = [s.strip() for s in exp.value] + if 'op' not in exp: + outer = ast.filter.Any + cond = ast.filter.Includes(*values) + elif exp.op == '~': + outer = ast.filter.Any + cond = ast.filter.Includes(*values, approx=True) + elif exp.op == '!': + outer = ast.filter.All + cond = ast.filter.Excludes(*values) + elif exp.op == '!~': + outer = ast.filter.All + cond = ast.filter.Excludes(*values, approx=True) + else: # prevented by grammar + raise errors.ParserError('Invalid operator ({})'.format(exp.op), exp) + tokens.append(outer(ns.bse.tag, ast.filter.Any(ns.bst.label, cond))) + + elif exp.getName() == 'continuous': # FIXME: simplify and adapt bsfs.query.ast.filter.Between accordingly! + lo, hi = None, None + lo_inc, hi_inc = False, False + predicate = None + if 'eq' in exp: + # equation style + predicate = self._abb2uri[exp.eq.predicate.lower()] + + if ('>' in exp.eq.cleft and '<' in exp.eq.cright) or \ + ('<' in exp.eq.cleft and '>' in exp.eq.cright) or \ + (exp.eq.cleft == '=' and exp.eq.cright == '='): + # x > pred < y or x < pred > y or x = pred = y + raise errors.ParserError('Cannot have two lower or two upper bounds', exp) + + if '>' in exp.eq.cleft: + hi = exp.eq.vleft + hi_inc = '=' in exp.eq.cleft + elif '<' in exp.eq.cleft: + lo = exp.eq.vleft + lo_inc = '=' in exp.eq.cleft + elif exp.eq.cleft == '=': + hi = lo = exp.eq.vleft + lo_inc = hi_inc = True + + if '>' in exp.eq.cright: + lo = exp.eq.vright + lo_inc = '=' in exp.eq.cright + elif '<' in exp.eq.cright: + hi = exp.eq.vright + hi_inc = '=' in exp.eq.cright + elif exp.eq.cright == '=': + hi = lo = exp.eq.vright + lo_inc = hi_inc = True + + elif 'range' in exp: # value in [lo:hi] + predicate = self._abb2uri[exp.range.predicate.lower()] + + if 'lo' in exp.range: + lo = exp.range.lo + lo_inc = exp.range.bopen == '[' + if 'hi' in exp.range: + hi = exp.range.hi + hi_inc = exp.range.bclose == ']' + + else: # prevented by grammar + raise errors.ParserError('Expression is neither a range nor an equation', exp) + + # interpret values + if predicate in self._DATETIME_PREDICATES: + + # turn into datetime + lo, lfmt = datefmt.guess_datetime(lo) if lo is not None else (None, None) + hi, hfmt = datefmt.guess_datetime(hi) if hi is not None else (None, None) + + if lo is None and hi is None: # prevented by grammar + raise errors.ParserError('At least one bound must be present', exp) + + # turn the query into the format lo <= pred < hi by adjusting the boundaries + if hi == lo and lo_inc and hi_inc: + # example: pred = 2012 -> 1.1.2012 <= pred < 1.1.2013 + hi = datefmt.increment(lo, lfmt) + lo_inc = True + hi_inc = False + else: + if lo is not None: + # example: pred >= 2012 -> pred >= 1.1.2012, 00:00 + lo = datefmt.increment(lo, lfmt) if not lo_inc else lo + lo_inc = True + + if hi is not None: + # example: pred <= 2012 -> pred < 1.1.2013, 00:00 + hi = datefmt.increment(hi, hfmt) if hi_inc else hi + hi_inc = False + + # build the ast node + if (lo is not None and lfmt.is_time()) or (hi is not None and hfmt.is_time()): + # time specification + + if (lo is not None and not lfmt.is_time()) or \ + (hi is not None and not hfmt.is_time()): + # lo/hi must both be time specifications + raise errors.ParserError('Both bounds must be a time specification', (lo, hi)) + + if lo is None: + # example: pred < 5 am -> 0 <= pred < 05:00 + lo = ttime.from_timestamp_loc(0) + lo_inc = True + + if hi is None: + # example: pred > 5 am -> 06:00 <= pred <= 24:00 + hi = ttime.from_timestamp_loc(3600 * 24) + hi_inc = True + + # Check consistency + if not (lo < hi or (lo == hi and lo_inc and hi_inc)): + raise errors.ParserError('Lower bound must not exceed upper bound', (lo, hi)) + + tokens.append( + ast.filter.Any(predicate, + ast.filter.Between(lo, hi, not lo_inc, not hi_inc))) + + else: # date specification + # Check consistency + lo = lo if lo is not None else datetime.min + hi = hi if hi is not None else datetime.max + + if not (lo < hi or (lo == hi and lo_inc and hi_inc)): + raise errors.ParserError('Lower bound must not exceed upper bound', (lo, hi)) + + tokens.append( + ast.filter.Any(predicate, + ast.filter.Between(lo, hi, not lo_inc, not hi_inc))) + + else: + # number predicate + lo = float(''.join(lo)) if lo is not None else float('-inf') + hi = float(''.join(hi)) if hi is not None else float('inf') + + # Check consistency + if not (lo < hi or (lo == hi and lo_inc and hi_inc)): + raise errors.ParserError('Lower bound must not exceed upper bound', (lo, hi)) + + # FIXME: mb/port: Three times the same code... optimize + tokens.append( + ast.filter.Any(predicate, + ast.filter.Between(lo, hi, not lo_inc, not hi_inc))) + + else: # prevented by grammar + raise errors.ParserError('Invalid expression', exp) + + return ast.filter.And(tokens) + +## EOF ## diff --git a/tagit/parsing/search.py b/tagit/parsing/search.py deleted file mode 100644 index 10d0e7c..0000000 --- a/tagit/parsing/search.py +++ /dev/null @@ -1,405 +0,0 @@ -"""User-specified search query parsing. - ->>> q = "has mime / tag in (november, october) / ! Apfel / time < 10.10.2004 / iso in (100, 200)" ->>> ast = ast_from_string(q) - -Part of the tagit module. -A copy of the license is provided with the project. -Author: Matthias Baumgartner, 2022 -""" -# standard imports -from datetime import datetime - -# external imports -from pyparsing import CaselessKeyword, Combine, Group, Optional, Or, Word, delimitedList, nums, oneOf, ParseException, Literal, QuotedString, alphanums, alphas8bit, punc8bit - -# tagit imports -from tagit.utils import errors, ttime - -# inner-module imports -from . import datefmt - -# exports -__all__ = ( - 'ast_from_string', - ) - -# constants -SEARCH_DELIM = '/' -VALUE_DELIM = ',' -DEFAULT_PREDICATE = 'tag' - - -## code ## - -class SearchParser(): - - # valid predicates per type - _PREDICATES_CATEGORICAL = None - _PREDICATES_CONTINUOUS = None - _PREDICATES_DATETIME = None - - # parsers - _CATEGORICAL = None - _CONTINUOUS = None - _EXISTENCE = None - _QUERY = None - _TAG = None - - def __init__(self, schema: bsfs.schema.Schema): - self.schema = schema - - @property - def schema(self) -> bsfs.schema.Schema: - return self._schema - - @schema.setter - def schema(self, schema: bsfs.schema.Schema): - self._schema = schema - self.build_parser() - - def build_parser(self): - """ - """ - # The *predicate* argument is for compatibility with predicate listener. - # It's not actually used here. - - # valid predicates per type, as supplied by tagit.library - # FIXME: - # * range / type constraints - # * how to filter predicates - # * distinguish between prefix and label - """ - Categorical: string, float, integer; labelled node (tag, group); maybe boolean - Continuous: float, integer - Datetime: datetime - Existencial: all of the above, particularly bool; unllabelled node (preview) - -> rfds:range - - > Target: Entity (allow others?) -> rfds:domain - > Require: searchable as specified in backend AND user-searchable as specified in frontend - """ - self._PREDICATES_CATEGORICAL = self.schema.predicates(searchable=True, range=self.schema.tm.categorical) # FIXME! - self._PREDICATES_CONTINUOUS = self.schema.predicates(searchable=True, range=self.schema.tm.numerical) # FIXME! - self._PREDICATES_DATETIME = self.schema.predicates(searchable=True, range=self.schema.tm.datetime) # FIXME! - - # terminal symbols - number = Group(Optional(oneOf('- +')) \ - + Combine(Word(nums) + Optional('.' + Optional(Word(nums))))) - words = QuotedString(quoteChar='"', escChar='\\') \ - ^ QuotedString(quoteChar="'", escChar='\\') \ - ^ Word(alphanums + alphas8bit + punc8bit + ' *#%&-.:;<=>?@^_`{}~') - # FIXME: allow escaped chars "( ) / , [ ]" - # FIXME: Non-ascii characters - - # predicates - predicate = Or([CaselessKeyword(p) for p in self._PREDICATES_CATEGORICAL]).setResultsName( - 'predicate') - date_predicate = Or([CaselessKeyword(p) for p in self._PREDICATES_DATETIME]).setResultsName( - 'predicate') - num_predicate = Or([CaselessKeyword(p) for p in self._PREDICATES_CONTINUOUS]).setResultsName( - 'predicate') - - # existence - """ - EXPR := has PREDICATE | has no PREDICATE - PREDICATE := [predicate] - """ - op = (CaselessKeyword('has') ^ CaselessKeyword('has no') ^ CaselessKeyword('has not')).setResultsName('op') - self._EXISTENCE = Group(op + predicate).setResultsName('existence') - - - # continuous - """ - EXPR := PREDICATE CMP VALUE | VALUE CMP PREDICATE CMP VALUE | PREDICATE OP RANGE - PREDICATE := [predicate] - CMP := < | <= | = | >= | > - OP := : | = | in | not in - RANGE := BOPEN VALUE RSEP VALUE BCLOSE | BOPEN RSEP VALUE BLOSE | BOPEN VALUE RSEP BCLOSE - BOPEN := ( | [ | ] - BCLOSE := ) | ] | [ - RSEP := : | - - VALUE := [digit] | [date] - """ - # range style - rsepn = oneOf(': -') - rsepd = Literal('-') - bclose = oneOf(') ] [').setResultsName('bclose') - bopen = oneOf('( [ ]').setResultsName('bopen') - op = Or([':', '=', 'in']).setResultsName('op') - datefmt = datefmt.parse_datetime.DATETIME - rngn = num_predicate + op + bopen + number('lo') + rsepn + number('hi') + bclose ^ \ - num_predicate + op + bopen + rsepn + number('hi') + bclose ^ \ - num_predicate + op + bopen + number('lo') + rsepn + bclose - rngd = date_predicate + op + bopen + datefmt('lo') + rsepd + datefmt('hi') + bclose ^ \ - date_predicate + op + bopen + rsepd + datefmt('hi') + bclose ^ \ - date_predicate + op + bopen + datefmt('lo') + rsepd + bclose - # equation style - cmp = oneOf('< <= = >= >').setResultsName('cmp') - eqn = num_predicate + cmp('cright') + number('vright') ^ \ - number('vleft') + cmp('cleft') + num_predicate ^ \ - number('vleft') + cmp('cleft') + num_predicate + cmp('cright') + number('vright') - eqd = date_predicate + cmp('cright') + datefmt('vright') ^ \ - datefmt('vleft') + cmp('cleft') + date_predicate ^ \ - datefmt('vleft') + cmp('cleft') + date_predicate + cmp('cright') + datefmt('vright') - # combined - self._CONTINUOUS = Group( - Group(eqn).setResultsName('eq') ^ - Group(eqd).setResultsName('eq') ^ - Group(rngn).setResultsName('range') ^ \ - Group(rngd).setResultsName('range') \ - ).setResultsName('continuous') - - - # categorical - """ - EXPR := PREDICATE OP VALUE | PREDICATE OP (VALUE) - PREDICATE := [predicate] - OP := : | = | in | not in | != | ~ | !~ - VALUE := TERM | VALUE, - TERM := [word] - """ - op = (CaselessKeyword('in') ^ CaselessKeyword('not in') ^ ':' ^ '=' ^ '!=' ^ '~' ^ '!~').setResultsName('op') - value = delimitedList(words, delim=VALUE_DELIM).setResultsName('value') - self._CATEGORICAL = Group(predicate + op + ('(' + value + ')' | value) ).setResultsName('categorical') - - - # tag shortcuts - """ - EXPR := OP VALUE | OP (VALUE) | VALUE | (VALUE) - OP := ! | ~ | !~ - VALUE := TERM | VALUE, - TERM := [word] - """ - op = oneOf('! ~ !~').setResultsName('op') - value = delimitedList(words, delim=VALUE_DELIM).setResultsName('value') - self._TAG = Group(Optional(op) + '(' + value + ')' ^ Optional(op) + value).setResultsName('tag') - - - # overall query - """ - QUERY := QUERY / QUERY | EXPR - """ - self._QUERY = delimitedList(self._EXISTENCE | self._CONTINUOUS | self._CATEGORICAL | self._TAG, delim=SEARCH_DELIM) - return self - - def __del__(self): - if self._QUERY is not None: # remove listener - try: - self.predicates.ignore(self.build_parser) - except ImportError: - # The import fails if python is shutting down. - # In that case, the ignore becomes unnecessary anyway. - pass - - def __call__(self, search): - # FIXME: mb/port/parsing - #if self._QUERY is None: - # # parsers were not initialized yet - # self.build_parser() - # # attach listener to receive future updates - # self.predicates.listen(self.build_parser) - # # FIXME: Additional filters would be handy - # #self.predicates.listen(self.build_parser, self.predicates.scope.library) - - try: - parsed = self._QUERY.parseString(search, parseAll=True) - except ParseException as e: - raise errors.ParserError('Cannot parse query', e) - - # convert to AST - tokens = [] - for exp in parsed: - if exp.getName() == 'existence': - if 'op' not in exp: # prevented by grammar - raise errors.ParserError('Missing operator', exp) - elif exp.op == 'has': - cond = ast.Existence() - elif exp.op in ('has no', 'has not'): - cond = ast.Inexistence() - else: # prevented by grammar - raise errors.ParserError('Invalid operator ({})'.format(exp.op), exp) - - tokens.append( - ast.Token(exp.predicate.lower(), cond)) - - elif exp.getName() == 'categorical': - values = [s.strip() for s in exp.value] - if 'op' not in exp: # prevented by grammar - raise errors.ParserError('Missing operator', exp) - elif exp.op in (':', '=', 'in'): - cond = ast.SetInclude(values) - elif exp.op in ('!=', 'not in'): - cond = ast.SetExclude(values) - elif exp.op == '~': - cond = ast.SetInclude(values, approximate=True) - elif exp.op == '!~': - cond = ast.SetExclude(values, approximate=True) - else: # prevented by grammar - raise errors.ParserError('Invalid operator ({})'.format(exp.op), exp) - - tokens.append( - ast.Token(exp.predicate.lower(), cond)) - - elif exp.getName() == 'tag': - values = [s.strip() for s in exp.value] - if 'op' not in exp: - cond = ast.SetInclude(values) - elif exp.op == '~': - cond = ast.SetInclude(values, approximate=True) - elif exp.op == '!': - cond = ast.SetExclude(values) - elif exp.op == '!~': - cond = ast.SetExclude(values, approximate=True) - else: # prevented by grammar - raise errors.ParserError('Invalid operator ({})'.format(exp.op), exp) - - tokens.append( - ast.Token(DEFAULT_PREDICATE, cond)) - - elif exp.getName() == 'continuous': - - lo, hi = None, None - lo_inc, hi_inc = False, False - predicate = None - - if 'eq' in exp: - # equation style - predicate = exp.eq.predicate.lower() - - if ('>' in exp.eq.cleft and '<' in exp.eq.cright) or \ - ('<' in exp.eq.cleft and '>' in exp.eq.cright) or \ - (exp.eq.cleft == '=' and exp.eq.cright == '='): - # x > pred < y or x < pred > y or x = pred = y - raise errors.ParserError('Cannot have two lower or two upper bounds', exp) - - if '>' in exp.eq.cleft: - hi = exp.eq.vleft - hi_inc = '=' in exp.eq.cleft - elif '<' in exp.eq.cleft: - lo = exp.eq.vleft - lo_inc = '=' in exp.eq.cleft - elif exp.eq.cleft == '=': - hi = lo = exp.eq.vleft - lo_inc = hi_inc = True - - if '>' in exp.eq.cright: - lo = exp.eq.vright - lo_inc = '=' in exp.eq.cright - elif '<' in exp.eq.cright: - hi = exp.eq.vright - hi_inc = '=' in exp.eq.cright - elif exp.eq.cright == '=': - hi = lo = exp.eq.vright - lo_inc = hi_inc = True - - elif 'range' in exp: # value in [lo:hi] - predicate = exp.range.predicate.lower() - - if 'lo' in exp.range: - lo = exp.range.lo - lo_inc = exp.range.bopen == '[' - if 'hi' in exp.range: - hi = exp.range.hi - hi_inc = exp.range.bclose == ']' - - else: # prevented by grammar - raise errors.ParserError('Expression is neither a range nor an equation', exp) - - # interpret values - if predicate in set([p.lower() for p in self._PREDICATES_DATETIME]): - - # turn into datetime - lo, lfmt = datefmt.guess_datetime(lo) if lo is not None else (None, None) - hi, hfmt = datefmt.guess_datetime(hi) if hi is not None else (None, None) - - if lo is None and hi is None: # prevented by grammar - raise errors.ParserError('At least one bound must be present', exp) - - # turn the query into the format lo <= pred < hi by adjusting the boundaries - if hi == lo and lo_inc and hi_inc: - # example: pred = 2012 -> 1.1.2012 <= pred < 1.1.2013 - hi = datefmt.increment(lo, lfmt) - lo_inc = True - hi_inc = False - else: - if lo is not None: - # example: pred >= 2012 -> pred >= 1.1.2012, 00:00 - lo = datefmt.increment(lo, lfmt) if not lo_inc else lo - lo_inc = True - - if hi is not None: - # example: pred <= 2012 -> pred < 1.1.2013, 00:00 - hi = datefmt.increment(hi, hfmt) if hi_inc else hi - hi_inc = False - - # build the ast node - if (lo is not None and lfmt.is_time()) or (hi is not None and hfmt.is_time()): - # time specification - - if (lo is not None and not lfmt.is_time()) or \ - (hi is not None and not hfmt.is_time()): - # lo/hi must both be time specifications - raise errors.ParserError('Both bounds must be a time specification', (lo, hi)) - - if lo is None: - # example: pred < 5 am -> 0 <= pred < 05:00 - lo = ttime.from_timestamp_loc(0) - lo_inc = True - - if hi is None: - # example: pred > 5 am -> 06:00 <= pred <= 24:00 - hi = ttime.from_timestamp_loc(3600 * 24) - hi_inc = True - - # Check consistency - if not (lo < hi or (lo == hi and lo_inc and hi_inc)): - raise errors.ParserError('Lower bound must not exceed upper bound', (lo, hi)) - - tokens.append( - ast.Token(predicate, ast.TimeRange(lo, hi, lo_inc, hi_inc))) - - else: # date specification - # Check consistency - lo = lo if lo is not None else datetime.min - hi = hi if hi is not None else datetime.max - - if not (lo < hi or (lo == hi and lo_inc and hi_inc)): - raise errors.ParserError('Lower bound must not exceed upper bound', (lo, hi)) - - tokens.append( - ast.Token(predicate, ast.Datetime(lo, hi, lo_inc, hi_inc))) - - else: - # number predicate - lo = float(''.join(lo)) if lo is not None else float('-inf') - hi = float(''.join(hi)) if hi is not None else float('inf') - - # Check consistency - if not (lo < hi or (lo == hi and lo_inc and hi_inc)): - raise errors.ParserError('Lower bound must not exceed upper bound', (lo, hi)) - - tokens.append( - ast.Token(predicate, ast.Continuous(lo, hi, lo_inc, hi_inc))) - - else: # prevented by grammar - raise errors.ParserError('Invalid expression', exp) - - return ast.AND(tokens) - - - -"""Default SearchParser instance. - -To produce an ast, call - ->>> ast_from_string(search) - -Convenience shortcut for - ->>> SearchParser().parse(search) - -""" -ast_from_string = SearchParser(predicates) - -## EOF ## diff --git a/tagit/parsing/sort.py b/tagit/parsing/sort.py index 8950613..75fa36c 100644 --- a/tagit/parsing/sort.py +++ b/tagit/parsing/sort.py @@ -12,13 +12,13 @@ from tagit.utils import errors, Struct # exports __all__ = ( - 'sort_from_string', + 'Sort', ) ## code ## -class SortParser(): +class Sort(): """Sort parser. A sort string can be as simple as a predicate, but also allows @@ -176,17 +176,4 @@ class SortParser(): else: return ast.Order(*tokens) -"""Default SortParser instance. - -To produce an ast, call - ->>> sort_from_string(sort) - -Convenience shortcut for - ->>> SortParser().parse(sort) - -""" -sort_from_string = SortParser(sortkeys) - ## EOF ## diff --git a/tagit/utils/__init__.py b/tagit/utils/__init__.py index 3f09078..16dcd4d 100644 --- a/tagit/utils/__init__.py +++ b/tagit/utils/__init__.py @@ -9,6 +9,7 @@ import typing # inner-module imports from . import bsfs +from . import namespaces as ns from . import time as ttime from .frame import Frame from .shared import * # FIXME: port properly diff --git a/tagit/utils/bsfs.py b/tagit/utils/bsfs.py index 0ab90a9..d80efe0 100644 --- a/tagit/utils/bsfs.py +++ b/tagit/utils/bsfs.py @@ -8,8 +8,16 @@ Author: Matthias Baumgartner, 2022 import typing # bsfs imports +from bsfs import schema, Open +from bsfs.query import ast +from bsfs.namespace import Namespace # exports -__all__: typing.Sequence[str] = [] +__all__: typing.Sequence[str] = ( + 'Namespace', + 'Open', + 'ast', + 'schema', + ) ## EOF ## diff --git a/tagit/utils/namespaces.py b/tagit/utils/namespaces.py new file mode 100644 index 0000000..dd26eef --- /dev/null +++ b/tagit/utils/namespaces.py @@ -0,0 +1,30 @@ +"""Default namespaces used throughout tagit. + +Part of the tagit module. +A copy of the license is provided with the project. +Author: Matthias Baumgartner, 2022 +""" +# standard imports +import typing + +# inner-module imports +from . import bsfs as _bsfs + +# constants +bse = _bsfs.Namespace('http://bsfs.ai/schema/Entity') +bsfs = _bsfs.Namespace('http://bsfs.ai/schema', fsep='/') +bsm = _bsfs.Namespace('http://bsfs.ai/schema/Meta') +bst = _bsfs.Namespace('http://bsfs.ai/schema/Tag') +xsd = _bsfs.Namespace('http://www.w3.org/2001/XMLSchema') + +# export +__all__: typing.Sequence[str] = ( + 'bse', + 'bsfs', + 'bsm', + 'xsd', + ) + +## EOF ## + + diff --git a/tagit/utils/shared.py b/tagit/utils/shared.py index 0d496ed..b5ab421 100644 --- a/tagit/utils/shared.py +++ b/tagit/utils/shared.py @@ -28,6 +28,7 @@ __all__: typing.Sequence[str] = ( 'is_list', 'magnitude_fmt', 'truncate_dir', + 'get_root', ) @@ -140,4 +141,15 @@ def fileopen(pth): except KeyError: warnings.warn('Unknown platform {}'.format(platform.system())) + +def get_root(obj): + """Traverse the widget tree upwards until the root is found.""" + while obj.parent is not None and obj.parent != obj.parent.parent: + if hasattr(obj, 'root') and obj.root is not None: + return obj.root + + obj = obj.parent + + return obj + ## EOF ## diff --git a/tagit/widgets/filter.py b/tagit/widgets/filter.py index 0152737..332ad34 100644 --- a/tagit/widgets/filter.py +++ b/tagit/widgets/filter.py @@ -23,7 +23,7 @@ import kivy.properties as kp # tagit imports from tagit import config #from tagit.parsing.search import ast, ast_to_string # FIXME: mb/port -from tagit.utils import errors +from tagit.utils import bsfs, errors # inner-module imports from .session import ConfigAwareMixin @@ -108,6 +108,10 @@ class Filter(BoxLayout, ConfigAwareMixin): ## exposed methods def get_query(self): + query = bsfs.ast.filter.And(self.t_head[:]) if len(self.t_head) > 0 else None + sort = None + return query, sort + # FIXME: mb/port.parsing query = ast.AND(self.t_head[:]) if len(self.t_head) else None # sort order is always set to False so that changing the sort order # won't trigger a new query which can be very expensive. The sort @@ -116,6 +120,8 @@ class Filter(BoxLayout, ConfigAwareMixin): return query, sort def abbreviate(self, token): + return 'T' + # FIXME: mb/port/parsing if token.predicate() == 'tag': return ','.join(list(token.condition())) elif token.predicate() == 'entity': @@ -162,8 +168,9 @@ class Filter(BoxLayout, ConfigAwareMixin): if self.changed: self.redraw() # issue search - if self.run_search: - self.root.trigger('Search') + # FIXME: mb/port/parsing + #if self.run_search: + # self.root.trigger('Search') def redraw(self): self.tokens.clear_widgets() diff --git a/tagit/widgets/session.py b/tagit/widgets/session.py index a7c7355..ca8c595 100644 --- a/tagit/widgets/session.py +++ b/tagit/widgets/session.py @@ -14,6 +14,7 @@ from kivy.uix.widget import Widget import kivy.properties as kp # tagit imports +from tagit import parsing from tagit.config.loader import load_settings #from tagit.storage.broker import Broker # FIXME: mb/port #from tagit.storage.loader import load_broker, load_log # FIXME: mb/port @@ -38,6 +39,9 @@ class Session(Widget): self.cfg = cfg self.storage = storage self.log = log + # derived members + self.filter_from_string = parsing.Filter(self.storage.schema) + #self.sort_from_string = parsing.Sort(self.storage.schema) # FIXME: mb/port/parsing def __enter__(self): return self diff --git a/test/parsing/test_filter.py b/test/parsing/test_filter.py new file mode 100644 index 0000000..c01c1bf --- /dev/null +++ b/test/parsing/test_filter.py @@ -0,0 +1,751 @@ +""" + +Part of the tagit test suite. +A copy of the license is provided with the project. +Author: Matthias Baumgartner, 2022 +""" +# standard imports +import unittest +from datetime import datetime + +# external imports +from pyparsing import ParseException + +# tagit imports +from tagit.utils import bsfs, errors, ns +from tagit.utils.bsfs import ast + +# objects to test +from tagit.parsing.filter import Filter + + +## code ## + +class TestFilterRange(unittest.TestCase): + longMessage = True + + def setUp(self): + #predicates.expose('mime', TestScope('attribute', 'mime'), 'Categorical') + #predicates.expose('iso', TestScope('attribute', 'iso'), 'Continuous', 'Categorical') + #predicates.expose('time', TestScope('generic', 't_image_create_loc'), 'TimeRange', 'Datetime') + #predicates.expose('tag', TestScope('generic', 'tag'), 'Categorical') + + #predicates.expose('mime', TestScope('attribute', 'mime'), 'Categorical') + #predicates.expose('rank', TestScope('attribute', 'rank'), 'Continuous') + #predicates.expose('iso', TestScope('attribute', 'iso'), 'Continuous', 'Categorical') + #predicates.expose('time', TestScope('generic', 't_image_create_loc'), 'TimeRange', 'Datetime') + #predicates.expose('tag', TestScope('generic', 'tag'), 'Categorical') + + + self.schema = bsfs.schema.from_string(''' + # common external prefixes + prefix rdfs: + prefix xsd: + + # common bsfs prefixes + prefix bsfs: + prefix bse: + + # nodes + bsfs:Entity rdfs:subClassOf bsfs:Node . + bsfs:Tag rdfs:subClassOf bsfs:Node . + + # literals + bsfs:Time rdfs:subClassOf bsfs:Literal . + xsd:string rdfs:subClassOf bsfs:Literal . + bsfs:Number rdfs:subClassOf bsfs:Literal . + xsd:integer rdfs:subClassOf bsfs:Number . + + # predicates + bse:mime rdfs:subClassOf bsfs:Predicate ; + rdfs:domain bsfs:Entity ; + rdfs:range xsd:string ; + bsfs:unique "true"^^xsd:boolean . + + bse:iso rdfs:subClassOf bsfs:Predicate ; + rdfs:domain bsfs:Entity ; + rdfs:range xsd:integer ; + bsfs:unique "true"^^xsd:boolean . + + bse:time rdfs:subClassOf bsfs:Predicate ; + rdfs:domain bsfs:Entity ; + rdfs:range bsfs:Time; + bsfs:unique "true"^^xsd:boolean . + + bse:tag rdfs:subClassOf bsfs:Predicate ; + rdfs:domain bsfs:Entity ; + rdfs:range bsfs:Tag ; + bsfs:unique "false"^^xsd:boolean . + + bse:rank rdfs:subClassOf bsfs:Predicate ; + rdfs:domain bsfs:Entity ; + rdfs:range xsd:integer ; + bsfs:unique "false"^^xsd:boolean . + + ''') + self.parse = Filter(self.schema) + + def _test_number(self, query, target): + predicate, condition = target + result = self.parse(query) + target = ast.filter.And(ast.filter.Any(predicate, condition)) + self.assertEqual(result, target, msg="in query '{}'".format(query)) + + def test_larger_than(self): + # larger than A (inclusive) + for editable in [ + # range + "{predicate} in [{num}:]", "{predicate} in [{num}:[", "{predicate} in [{num}:)", + "{predicate} : [{num}:]", "{predicate} : [{num}:[", "{predicate} : [{num}:)", + "{predicate} = [{num}:]", "{predicate} = [{num}:[", "{predicate} = [{num}:)", + ]: + # positive + self._test_number(editable.format(num=1.23, predicate='iso'), + (ns.bse.iso, ast.filter.GreaterThan(1.23, False))) + # negative + self._test_number(editable.format(num=-1.23, predicate='iso'), + (ns.bse.iso, ast.filter.GreaterThan(-1.23, False))) + + for editable in [ + # range + "{predicate} in [{num}-]", "{predicate} in [{num}-[", "{predicate} in [{num}-)", + "{predicate} : [{num}-]", "{predicate} : [{num}-[", "{predicate} : [{num}-)", + "{predicate} = [{num}-]", "{predicate} = [{num}-[", "{predicate} = [{num}-)", + # equation + "{predicate} >= {num}", "{num} <= {predicate}", + ]: + # positive + self._test_number(editable.format(num=1.23, predicate='iso'), + (ns.bse.iso, ast.filter.GreaterThan(1.23, False))) + # negative + self._test_number(editable.format(num=-1.23, predicate='iso'), + (ns.bse.iso, ast.filter.GreaterThan(-1.23, False))) + # FIXME: date + #self._test_number(editable.format(predicate='time', num="30.04.2012, 13:18"), + # ('time', ast.Datetime(datetime(2012, 4, 30, 13, 18), datetime.max, True, False))) + + # larger than A (exclusive) + for editable in [ + # range / bracket + "{predicate} in ]{num}:]", "{predicate} in ]{num}:[", "{predicate} in ]{num}:)", + "{predicate} : ]{num}:]", "{predicate} : ]{num}:[", "{predicate} : ]{num}:)", + "{predicate} = ]{num}:]", "{predicate} = ]{num}:[", "{predicate} = ]{num}:)", + # range / parenthesis + "{predicate} in ({num}:]", "{predicate} in ({num}:[", "{predicate} in ({num}:)", + "{predicate} : ({num}:]", "{predicate} : ({num}:[", "{predicate} : ({num}:)", + "{predicate} = ({num}:]", "{predicate} = ({num}:[", "{predicate} = ({num}:)", + ]: + # positive + self._test_number(editable.format(num=1.23, predicate='iso'), + (ns.bse.iso, ast.filter.GreaterThan(1.23, True))) + # negative + self._test_number(editable.format(num=-1.23, predicate='iso'), + (ns.bse.iso, ast.filter.GreaterThan(-1.23, True))) + + for editable in [ + # range / bracket + "{predicate} in ]{num}-]", "{predicate} in ]{num}-[", "{predicate} in ]{num}-)", + "{predicate} : ]{num}-]", "{predicate} : ]{num}-[", "{predicate} : ]{num}-)", + "{predicate} = ]{num}-]", "{predicate} = ]{num}-[", "{predicate} = ]{num}-)", + # range / parenthesis + "{predicate} in ({num}-]", "{predicate} in ({num}-[", "{predicate} in ({num}-)", + "{predicate} : ({num}-]", "{predicate} : ({num}-[", "{predicate} : ({num}-)", + "{predicate} = ({num}-]", "{predicate} = ({num}-[", "{predicate} = ({num}-)", + # equation + "{predicate} > {num}", "{num} < {predicate}", + ]: + # positive + self._test_number(editable.format(num=1.23, predicate='iso'), + (ns.bse.iso, ast.filter.GreaterThan(1.23, True))) + # negative + self._test_number(editable.format(num=-1.23, predicate='iso'), + (ns.bse.iso, ast.filter.GreaterThan(-1.23, True))) + # FIXME: date + #self._test_number(editable.format(predicate='time', num="30.04.2012, 13:18"), + # ('time', ast.Datetime(datetime(2012, 4, 30, 13, 19), datetime.max, True, False))) + + def test_smaller_than(self): + # smaller than B (inclusive) + for editable in [ + # range + "{predicate} in [:{num}]", "{predicate} in (:{num}]", "{predicate} in ]:{num}]", + "{predicate} : [:{num}]", "{predicate} : (:{num}]", "{predicate} : ]:{num}]", + "{predicate} = [:{num}]", "{predicate} = (:{num}]", "{predicate} = ]:{num}]", + ]: + # positives + self._test_number(editable.format(num=1.23, predicate='iso'), + (ns.bse.iso, ast.filter.LessThan(1.23, False))) + # negatives + self._test_number(editable.format(num=-1.23, predicate='iso'), + (ns.bse.iso, ast.filter.LessThan(-1.23, False))) + + for editable in [ + # range + "{predicate} in [-{num}]", "{predicate} in (-{num}]", "{predicate} in ]-{num}]", + "{predicate} : [-{num}]", "{predicate} : (-{num}]", "{predicate} : ]-{num}]", + "{predicate} = [-{num}]", "{predicate} = (-{num}]", "{predicate} = ]-{num}]", + # equation + "{predicate} <={num}", "{num} >= {predicate}", + ]: + # positives + self._test_number(editable.format(num=1.23, predicate='iso'), + (ns.bse.iso, ast.filter.LessThan(1.23, False))) + # negatives + self._test_number(editable.format(num=-1.23, predicate='iso'), + (ns.bse.iso, ast.filter.LessThan(-1.23, False))) + # FIXME: date + #self._test_number(editable.format(predicate='time', num="30.04.2012, 13:18"), + # ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 13, 19), False, False))) + + # smaller than B (exclusive) + for editable in [ + # range / bracket + "{predicate} in [:{num}[", "{predicate} in (:{num}[", "{predicate} in ]:{num}[", + "{predicate} : [:{num}[", "{predicate} : (:{num}[", "{predicate} : ]:{num}[", + "{predicate} = [:{num}[", "{predicate} = (:{num}[", "{predicate} = ]:{num}[", + # range / parenthesis + "{predicate} in [:{num})", "{predicate} in (:{num})", "{predicate} in ]:{num})", + "{predicate} : [:{num})", "{predicate} : (:{num})", "{predicate} : ]:{num})", + "{predicate} = [:{num})", "{predicate} = (:{num})", "{predicate} = ]:{num})", + ]: + # positives + self._test_number(editable.format(num=1.23, predicate='iso'), + (ns.bse.iso, ast.filter.LessThan(1.23, True))) + # negatives + self._test_number(editable.format(num=-1.23, predicate='iso'), + (ns.bse.iso, ast.filter.LessThan(-1.23, True))) + + for editable in [ + # range / bracket + "{predicate} in [-{num}[", "{predicate} in (-{num}[", "{predicate} in ]-{num}[", + "{predicate} : [-{num}[", "{predicate} : (-{num}[", "{predicate} : ]-{num}[", + "{predicate} = [-{num}[", "{predicate} = (-{num}[", "{predicate} = ]-{num}[", + # range / parenthesis + "{predicate} in [-{num})", "{predicate} in (-{num})", "{predicate} in ]-{num})", + "{predicate} : [-{num})", "{predicate} : (-{num})", "{predicate} : ]-{num})", + "{predicate} = [-{num})", "{predicate} = (-{num})", "{predicate} = ]-{num})", + # equation + "{predicate} <{num}", "{num} > {predicate}", + ]: + # positives + self._test_number(editable.format(num=1.23, predicate='iso'), + (ns.bse.iso, ast.filter.LessThan(1.23, True))) + # negatives + self._test_number(editable.format(num=-1.23, predicate='iso'), + (ns.bse.iso, ast.filter.LessThan(-1.23, True))) + # FIXME: date + #self._test_number(editable.format(predicate='time', num="30.04.2012, 13:18"), + # ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 13, 18), False, False))) + + def test_between(self): + # between A and B (including A, including B) + for editable in [ + # range + "{predicate} in [{numA}:{numB}]", "{predicate} : [{numA}:{numB}]", "{predicate} = [{numA}:{numB}]", + ]: + # positives + self._test_number(editable.format(predicate='iso', numA=1.23, numB=4.56), + (ns.bse.iso, ast.filter.Between(1.23, 4.56, False, False))) + # negatives + self._test_number(editable.format(predicate='iso', numA=-4.56, numB=-1.23), + (ns.bse.iso, ast.filter.Between(-4.56, -1.23, False, False))) + # mixed + self._test_number(editable.format(predicate='iso', numA=-1.23, numB=4.56), + (ns.bse.iso, ast.filter.Between(-1.23, 4.56, False, False))) + + for editable in [ + # range + "{predicate} in [{numA}-{numB}]", "{predicate} : [{numA}-{numB}]", "{predicate} = [{numA}-{numB}]", + # equation + "{numA} <= {predicate} <= {numB}" + ]: + # positives + self._test_number(editable.format(predicate='iso', numA=1.23, numB=4.56), + (ns.bse.iso, ast.filter.Between(1.23, 4.56, False, False))) + # negatives + self._test_number(editable.format(predicate='iso', numA=-4.56, numB=-1.23), + (ns.bse.iso, ast.filter.Between(-4.56, -1.23, False, False))) + # mixed + self._test_number(editable.format(predicate='iso', numA=-1.23, numB=4.56), + (ns.bse.iso, ast.filter.Between(-1.23, 4.56, False, False))) + # FIXME: date + #self._test_number(editable.format(predicate='time', numA="30.04.2012, 13:18", numB="13.6.2014, 18:27"), + # ('time', ast.Datetime(datetime(2012, 4, 30, 13, 18), datetime(2014, 6, 13, 18, 28), True, False))) + + # between A and B (including A, excluding B) + for editable in [ + # range + "{predicate} in [{numA}:{numB})", "{predicate} in [{numA}:{numB}[", + "{predicate} : [{numA}:{numB})", "{predicate} : [{numA}:{numB}[", + "{predicate} = [{numA}:{numB})", "{predicate} = [{numA}:{numB}[", + ]: + # positives + self._test_number(editable.format(predicate='iso', numA=1.23, numB=4.56), + (ns.bse.iso, ast.filter.Between(1.23, 4.56, False, True))) + # negatives + self._test_number(editable.format(predicate='iso', numA=-4.56, numB=-1.23), + (ns.bse.iso, ast.filter.Between(-4.56, -1.23, False, True))) + # mixed + self._test_number(editable.format(predicate='iso', numA=-1.23, numB=4.56), + (ns.bse.iso, ast.filter.Between(-1.23, 4.56, False, True))) + + for editable in [ + # range + "{predicate} in [{numA}-{numB})", "{predicate} in [{numA}-{numB}[", + "{predicate} : [{numA}-{numB})", "{predicate} : [{numA}-{numB}[", + "{predicate} = [{numA}-{numB})", "{predicate} = [{numA}-{numB}[", + # equation + "{numA} <= {predicate} < {numB}", + ]: + # positives + self._test_number(editable.format(predicate='iso', numA=1.23, numB=4.56), + (ns.bse.iso, ast.filter.Between(1.23, 4.56, False, True))) + # negatives + self._test_number(editable.format(predicate='iso', numA=-4.56, numB=-1.23), + (ns.bse.iso, ast.filter.Between(-4.56, -1.23, False, True))) + # mixed + self._test_number(editable.format(predicate='iso', numA=-1.23, numB=4.56), + (ns.bse.iso, ast.filter.Between(-1.23, 4.56, False, True))) + # FIXME: date + #self._test_number(editable.format(predicate='time', numA="30.04.2012, 13:18", numB="13.6.2014, 18:27"), + # ('time', ast.Datetime(datetime(2012, 4, 30, 13, 18), datetime(2014, 6, 13, 18, 27), True, False))) + + # between A and B (excluding A, including B) + for editable in [ + # range + "{predicate} in ({numA}:{numB}]", "{predicate} in ]{numA}:{numB}]", + "{predicate} : ({numA}:{numB}]", "{predicate} : ]{numA}:{numB}]", + "{predicate} = ({numA}:{numB}]", "{predicate} = ]{numA}:{numB}]", + ]: + # positives + self._test_number(editable.format(predicate='iso', numA=1.23, numB=4.56), + (ns.bse.iso, ast.filter.Between(1.23, 4.56, True, False))) + # negatives + self._test_number(editable.format(predicate='iso', numA=-4.56, numB=-1.23), + (ns.bse.iso, ast.filter.Between(-4.56, -1.23, True, False))) + # mixed + self._test_number(editable.format(predicate='iso', numA=-1.23, numB=4.56), + (ns.bse.iso, ast.filter.Between(-1.23, 4.56, True, False))) + + for editable in [ + # range + "{predicate} in ({numA}-{numB}]", "{predicate} in ]{numA}-{numB}]", + "{predicate} : ({numA}-{numB}]", "{predicate} : ]{numA}-{numB}]", + "{predicate} = ({numA}-{numB}]", "{predicate} = ]{numA}-{numB}]", + # equation + "{numA} < {predicate} <= {numB}", + ]: + # positives + self._test_number(editable.format(predicate='iso', numA=1.23, numB=4.56), + (ns.bse.iso, ast.filter.Between(1.23, 4.56, True, False))) + # negatives + self._test_number(editable.format(predicate='iso', numA=-4.56, numB=-1.23), + (ns.bse.iso, ast.filter.Between(-4.56, -1.23, True, False))) + # mixed + self._test_number(editable.format(predicate='iso', numA=-1.23, numB=4.56), + (ns.bse.iso, ast.filter.Between(-1.23, 4.56, True, False))) + # FIXME: date + #self._test_number(editable.format(predicate='time', numA="30.04.2012, 13:18", numB="13.6.2014, 18:27"), + # ('time', ast.Datetime(datetime(2012, 4, 30, 13, 19), datetime(2014, 6, 13, 18, 28), True, False))) + + # between A and B (excluding A, excluding B) + for editable in [ + "{predicate} in ({numA}:{numB})", "{predicate} in ]{numA}:{numB}[", + "{predicate} : ({numA}:{numB})", "{predicate} : ]{numA}:{numB}[", + "{predicate} = ({numA}:{numB})", "{predicate} = ]{numA}:{numB}[", + ]: + # positives + self._test_number(editable.format(predicate='iso', numA=1.23, numB=4.56), + (ns.bse.iso, ast.filter.Between(1.23, 4.56, True, True))) + # negatives + self._test_number(editable.format(predicate='iso', numA=-4.56, numB=-1.23), + (ns.bse.iso, ast.filter.Between(-4.56, -1.23, True, True))) + # mixed + self._test_number(editable.format(predicate='iso', numA=-1.23, numB=4.56), + (ns.bse.iso, ast.filter.Between(-1.23, 4.56, True, True))) + + for editable in [ + "{predicate} in ({numA}-{numB})", "{predicate} in ]{numA}-{numB}[", + "{predicate} : ({numA}-{numB})", "{predicate} : ]{numA}-{numB}[", + "{predicate} = ({numA}-{numB})", "{predicate} = ]{numA}-{numB}[", + # equation + "{numA} < {predicate} < {numB}", + ]: + # positives + self._test_number(editable.format(predicate='iso', numA=1.23, numB=4.56), + (ns.bse.iso, ast.filter.Between(1.23, 4.56, True, True))) + # negatives + self._test_number(editable.format(predicate='iso', numA=-4.56, numB=-1.23), + (ns.bse.iso, ast.filter.Between(-4.56, -1.23, True, True))) + # mixed + self._test_number(editable.format(predicate='iso', numA=-1.23, numB=4.56), + (ns.bse.iso, ast.filter.Between(-1.23, 4.56, True, True))) + # FIXME: date + #self._test_number(editable.format(predicate='time', numA="30.04.2012, 13:18", numB="13.6.2014, 18:27"), + # ('time', ast.Datetime(datetime(2012, 4, 30, 13, 19), datetime(2014, 6, 13, 18, 27), True, False))) + + def test_equal(self): + # equal to A + for editable in [ + # range + "{predicate} in [{num}:{num}]", "{predicate} : [{num}:{num}]", "{predicate} = [{num}:{num}]", + ]: + # positives + self._test_number(editable.format(predicate='iso', num=1.23), + (ns.bse.iso, ast.filter.Equals(1.23))) + # negatives + self._test_number(editable.format(predicate='iso', num=-1.23), + (ns.bse.iso, ast.filter.Equals(-1.23))) + + for editable in [ + # range + "{predicate} in [{num}-{num}]", "{predicate} : [{num}-{num}]", "{predicate} = [{num}-{num}]", + # equation + "{predicate} = {num}", "{num} = {predicate}", + ]: + # positives + self._test_number(editable.format(predicate='iso', num=1.23), + (ns.bse.iso, ast.filter.Equals(1.23))) + # negatives + self._test_number(editable.format(predicate='iso', num=-1.23), + (ns.bse.iso, ast.filter.Equals(-1.23))) + # FIXME: date + #self._test_number(editable.format(predicate='time', num="30.04.2012, 13:18"), + # ('time', ast.Datetime(datetime(2012, 4, 30, 13, 18), datetime(2012, 4, 30, 13, 19), True, False))) + + def test_dates(self): + raise NotImplementedError() # FIXME + self._test_number("{predicate} < {num}".format(predicate='time', num="2012"), + ('time', ast.Datetime(datetime.min, datetime(2012, 1, 1), False, False))) + self._test_number("{predicate} < {num}".format(predicate='time', num="2012.04"), + ('time', ast.Datetime(datetime.min, datetime(2012, 4, 1), False, False))) + self._test_number("{predicate} < {num}".format(predicate='time', num="2012.04.30"), + ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30), False, False))) + self._test_number("{predicate} < {num}".format(predicate='time', num="2012.04.30, 3 pm"), + ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 15), False, False))) + self._test_number("{predicate} < {num}".format(predicate='time', num="2012.04.30, 15:34"), + ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 15, 34), False, False))) + self._test_number("{predicate} < {num}".format(predicate='time', num="2012.04.30, 15:34:12"), + ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 15, 34, 12), False, False))) + self._test_number("{predicate} < {num}".format(predicate='time', num="2012.04.30, 15:34:12.98"), + ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 15, 34, 12, 980000), False, False))) + + self._test_number("{predicate} <= {num}".format(predicate='time', num="2012"), + ('time', ast.Datetime(datetime.min, datetime(2013, 1, 1), False, False))) + self._test_number("{predicate} <= {num}".format(predicate='time', num="2012.04"), + ('time', ast.Datetime(datetime.min, datetime(2012, 5, 1), False, False))) + self._test_number("{predicate} <= {num}".format(predicate='time', num="2012.04.30"), + ('time', ast.Datetime(datetime.min, datetime(2012, 5, 1), False, False))) + self._test_number("{predicate} <= {num}".format(predicate='time', num="2012.04.30, 3 pm"), + ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 16), False, False))) + self._test_number("{predicate} <= {num}".format(predicate='time', num="2012.04.30, 15:34"), + ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 15, 35), False, False))) + self._test_number("{predicate} <= {num}".format(predicate='time', num="2012.04.30, 15:34:12"), + ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 15, 34, 13), False, False))) + self._test_number("{predicate} <= {num}".format(predicate='time', num="2012.04.30, 15:34:12.98"), + ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 15, 34, 12, 980001), False, False))) + + def test_timerange(self): + raise NotImplementedError() # FIXME + self._test_number("{predicate} < {num}".format(predicate='time', num="15:34"), + ('time', ast.TimeRange(datetime.utcfromtimestamp(0.0), datetime(1970, 1, 1, 15, 34), True, False))) + self._test_number("{predicate} <= {num}".format(predicate='time', num="15:34"), + ('time', ast.TimeRange(datetime.utcfromtimestamp(0.0), datetime(1970, 1, 1, 15, 35), True, False))) + self._test_number("{predicate} = {num}".format(predicate='time', num="15:34"), + ('time', ast.TimeRange(datetime(1970, 1, 1, 15, 34), datetime(1970, 1, 1, 15, 35), True, False))) + self._test_number("{predicate} > {num}".format(predicate='time', num="15:34"), + ('time', ast.TimeRange(datetime(1970, 1, 1, 15, 35), datetime(1970, 1, 2), True, True))) + self._test_number("{predicate} >= {num}".format(predicate='time', num="15:34"), + ('time', ast.TimeRange(datetime(1970, 1, 1, 15, 34), datetime(1970, 1, 2), True, True))) + + self._test_number("{numA} <= {predicate} <= {numB}".format(predicate='time', numA="12:34", numB="15:28"), + ('time', ast.TimeRange(datetime(1970, 1, 1, 12, 34), datetime(1970, 1, 1, 15, 29), True, False))) + self._test_number("{numA} <= {predicate} < {numB}".format(predicate='time', numA="12:34", numB="15:28"), + ('time', ast.TimeRange(datetime(1970, 1, 1, 12, 34), datetime(1970, 1, 1, 15, 28), True, False))) + self._test_number("{numA} < {predicate} <= {numB}".format(predicate='time', numA="12:34", numB="15:28"), + ('time', ast.TimeRange(datetime(1970, 1, 1, 12, 35), datetime(1970, 1, 1, 15, 29), True, False))) + self._test_number("{numA} < {predicate} < {numB}".format(predicate='time', numA="12:34", numB="15:28"), + ('time', ast.TimeRange(datetime(1970, 1, 1, 12, 35), datetime(1970, 1, 1, 15, 28), True, False))) + + def test_special(self): + # special cases: explicit plus sign + self._test_number("{predicate} in [+1.23-+4.56]".format(predicate='iso'), + (ns.bse.iso, ast.filter.Between(1.23, 4.56, False, False))) + self._test_number("{predicate} in [-+4.56]".format(predicate='iso'), + (ns.bse.iso, ast.filter.LessThan(4.56, False))) + + def test_errors(self): + # parse errors + for editable in [ + # equal with exclusive + "{predicate} in ({num}:{num})", "{predicate} in ({num}-{num})", + "{predicate} in ({num}:{num}[", "{predicate} in ({num}-{num}[", + "{predicate} in ]{num}:{num})", "{predicate} in ]{num}-{num})", + "{predicate} in ]{num}:{num}[", "{predicate} in ]{num}-{num}[", + # invalid parentesis + "{predicate} in ){num}:{num}(", + # misc errors + # FIXME: Currently all special characters are allowed as categorical value. + # If this changes, don't forget to enable the tests below. + #"{predicate} in [{num}{num}]", + #"{predicate} [{num}:{num}:{num}]", + #"{predicate} = ({num})", + #"{predicate} = {num})", + ]: + self.assertRaises(errors.ParserError, self.parse, + editable.format(predicate='iso', num=1.23)) + + for editable in [ + "{predicate} in [{numA}:{numB}]", "{predicate} : [{numA}:{numB}]", "{predicate} = [{numA}:{numB}]", + "{predicate} in ]{numA}:{numB}]", "{predicate} : ]{numA}:{numB}]", "{predicate} = ]{numA}:{numB}]", + "{predicate} in [{numA}:{numB}[", "{predicate} : [{numA}:{numB}[", "{predicate} = [{numA}:{numB}[", + "{predicate} in ({numA}:{numB}]", "{predicate} : ({numA}:{numB}]", "{predicate} = ({numA}:{numB}]", + "{predicate} in [{numA}:{numB})", "{predicate} : [{numA}:{numB})", "{predicate} = [{numA}:{numB})", + "{predicate} in ]{numA}:{numB}[", "{predicate} : ]{numA}:{numB}[", "{predicate} = ]{numA}:{numB}[", + "{predicate} in ]{numA}:{numB})", "{predicate} : ]{numA}:{numB})", "{predicate} = ]{numA}:{numB})", + "{predicate} in ({numA}:{numB}[", "{predicate} : ({numA}:{numB}[", "{predicate} = ({numA}:{numB}[", + "{predicate} in ({numA}:{numB})", "{predicate} : ({numA}:{numB})", "{predicate} = ({numA}:{numB})", + "{numA} < {predicate} < {numB}", + "{numA} <= {predicate} < {numB}", + "{numA} < {predicate} <= {numB}", + ]: + self.assertRaises(errors.ParserError, self.parse, + editable.format(predicate='iso', numA=4.56, numB=1.23)) + # FIXME: + #self.assertRaises(errors.ParserError, self.parse, + # editable.format(predicate='time', numA="17:35", numB="10:55")) + #self.assertRaises(errors.ParserError, self.parse, + # editable.format(predicate='time', numA="18.12.2035", numB="5.7.1999")) + + raise NotImplementedError() # FIXME + # special cases: empty range with boundary + self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, + "{predicate} in [:]".format(predicate='iso')) + self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, + "{predicate} in (:[".format(predicate='iso')) + self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, + "{predicate} in ]:)".format(predicate='iso')) + self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, + "{predicate} in ".format(predicate='iso')) + # misc + self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, + "{predicate} in [{num}{num}]".format(predicate='iso', num=1.23)) + self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, + "{predicate} [{num}:{num}:{num}]".format(predicate='iso', num=1.23)) + self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, + "{predicate} = ({num})".format(predicate='iso', num=1.23)) + self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, + "{predicate} = ({num}".format(predicate='iso', num=1.23), dict(parseAll=True)) + self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, + "{predicate} = {num})".format(predicate='iso', num=1.23), dict(parseAll=True)) + # range errors + self.assertRaises(errors.ParserError, self.parse, "100 >= iso < 200") + self.assertRaises(errors.ParserError, self.parse, "100 > iso < 200") + self.assertRaises(errors.ParserError, self.parse, "100 > iso <= 200") + self.assertRaises(errors.ParserError, self.parse, "100 >= iso <= 200") + self.assertRaises(errors.ParserError, self.parse, "100 = iso = 200") + # time/date mixture errors + self.assertRaises(errors.ParserError, self.parse, "12:45 < time < 17.5.2004") + self.assertRaises(errors.ParserError, self.parse, "17.5.2004 < time < 12:45") + # date/int mixture errors + self.assertRaises(errors.ParserError, self.parse, "17.5.2004 < time < 1245") + # 1245 is interpreted as the year + #self.assertRaises(errors.ParserError, self.parse, "1245 < time < 17.5.2004") + # time/int mixture errors + self.assertRaises(errors.ParserError, self.parse, "17:12 < time < 1245") + self.assertRaises(errors.ParserError, self.parse, "1712 < time < 12:45") + + # empty query + self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, "") + + + + + def _test(self, query, target): + result = self.parse(query) + target = ast.filter.And(target) + self.assertEqual(result, target, msg="in query '{}'".format(query)) + + def test_parse_existence(self): + self._test('has mime', + ast.filter.Has(ns.bse.mime)) + self._test('has no mime', + ast.filter.Not(ast.filter.Has(ns.bse.mime))) + self._test('has not mime', + ast.filter.Not(ast.filter.Has(ns.bse.mime))) + + def test_parse_categorical(self): + # positive + self._test("iso in 100, 200, 500", + ast.filter.Any(ns.bse.iso, ast.filter.Includes('100', '200', '500'))) + self._test("iso in (100, 200)", + ast.filter.Any(ns.bse.iso, ast.filter.Includes('100', '200'))) + self._test("iso = (100, 200)", + ast.filter.Any(ns.bse.iso, ast.filter.Includes('100', '200'))) + # FIXME! + #self._test("iso = 100, 200", + # ast.filter.Any(ns.bse.iso, ast.filter.Includes('100', '200'))) + self._test("iso : (100, 200)", + ast.filter.Any(ns.bse.iso, ast.filter.Includes('100', '200'))) + self._test("iso : 100, 200", + ast.filter.Any(ns.bse.iso, ast.filter.Includes('100', '200'))) + self._test("iso:(100,200)", + ast.filter.Any(ns.bse.iso, ast.filter.Includes('100', '200'))) + self._test("iso in (100,200)", + ast.filter.Any(ns.bse.iso, ast.filter.Includes('100', '200'))) + self._test("iso in 100,200", + ast.filter.Any(ns.bse.iso, ast.filter.Includes('100', '200'))) + self._test("iso ~ (100,200)", + ast.filter.Any(ns.bse.iso, ast.filter.Includes('100', '200', approx=True))) + self._test("iso ~ 100,200", + ast.filter.Any(ns.bse.iso, ast.filter.Includes('100', '200', approx=True))) + + # negative + self._test("iso not in 100,200", + ast.filter.All(ns.bse.iso, ast.filter.Excludes('100', '200'))) + self._test("iso not in (100, 200)", + ast.filter.All(ns.bse.iso, ast.filter.Excludes('100', '200'))) + self._test("iso != 100,200", + ast.filter.All(ns.bse.iso, ast.filter.Excludes('100', '200'))) + self._test("iso != (100, 200)", + ast.filter.All(ns.bse.iso, ast.filter.Excludes('100', '200'))) + self._test("iso !~ 100,200", + ast.filter.All(ns.bse.iso, ast.filter.Excludes('100', '200', approx=True))) + self._test("iso !~ (100, 200)", + ast.filter.All(ns.bse.iso, ast.filter.Excludes('100', '200', approx=True))) + + # one value + self._test("mime : text", + ast.filter.Any(ns.bse.mime, ast.filter.Includes('text'))) + self._test("mime in text", + ast.filter.Any(ns.bse.mime, ast.filter.Includes('text'))) + self._test("mime = text", + ast.filter.Any(ns.bse.mime, ast.filter.Includes('text'))) + self._test("mime ~ text", + ast.filter.Any(ns.bse.mime, ast.filter.Includes('text', approx=True))) + self._test("mime != text", + ast.filter.All(ns.bse.mime, ast.filter.Excludes('text'))) + self._test("mime not in text", + ast.filter.All(ns.bse.mime, ast.filter.Excludes('text'))) + self._test("mime !~ text", + ast.filter.All(ns.bse.mime, ast.filter.Excludes('text', approx=True))) + + # expressions with slash and comma + self._test('mime : "text"', + ast.filter.Any(ns.bse.mime, ast.filter.Includes('text'))) + self._test('mime : "text", "plain"', + ast.filter.Any(ns.bse.mime, ast.filter.Includes('text', 'plain'))) + self._test('mime : "text, plain"', + ast.filter.Any(ns.bse.mime, ast.filter.Includes('text, plain'))) + self._test('mime ~ "text/plain"', + ast.filter.Any(ns.bse.mime, ast.filter.Includes('text/plain', approx=True))) + self._test('mime = ("text/plain", "image/jpeg")', + ast.filter.Any(ns.bse.mime, ast.filter.Includes('text/plain', 'image/jpeg'))) + + def test_parse_tag(self): + # only tag: tag, tags, (tag), (tags) + self._test("foo", + ast.filter.Any(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Equals('foo')))) + self._test("(foo)", + ast.filter.Any(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Equals('foo')))) + self._test("foo, bar", + ast.filter.Any(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Includes('foo', 'bar')))) + self._test("foo,bar", + ast.filter.Any(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Includes('foo', 'bar')))) + self._test("(foo, bar,foobar)", + ast.filter.Any(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Includes('foo', 'bar', 'foobar')))) + + # op and tag: !tag, ~tag, !~tag + self._test("~foo", + ast.filter.Any(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Substring('foo')))) + self._test("~ foo", + ast.filter.Any(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Substring('foo')))) + self._test("!foo", + ast.filter.All(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Not(ast.filter.Equals('foo'))))) + self._test("! foo", + ast.filter.All(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Not(ast.filter.Equals('foo'))))) + self._test("!~foo", + ast.filter.All(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Not(ast.filter.Substring('foo'))))) + self._test("!~ foo", + ast.filter.All(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Not(ast.filter.Substring('foo'))))) + + # op and list: ! (tags), ~tags, ... + self._test("~ foo, bar", + ast.filter.Any(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Includes('foo', 'bar', approx=True)))) + self._test("~foo, bar", + ast.filter.Any(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Includes('foo', 'bar', approx=True)))) + self._test("~ (foo, bar)", + ast.filter.Any(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Includes('foo', 'bar', approx=True)))) + self._test("! foo, bar", + ast.filter.All(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Excludes('foo', 'bar')))) + self._test("! (foo, bar)", + ast.filter.All(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Excludes('foo', 'bar')))) + self._test("! (foo,bar)", + ast.filter.All(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Excludes('foo', 'bar')))) + self._test("!~ foo, bar", + ast.filter.All(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Excludes('foo', 'bar', approx=True)))) + self._test("!~ (foo, bar)", + ast.filter.All(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Excludes('foo', 'bar', approx=True)))) + self._test("!~(foo,bar)", + ast.filter.All(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Excludes('foo', 'bar', approx=True)))) + + def test_parse_query(self): + # simple query + self.assertEqual(self.parse('foo / bar'), ast.filter.And( + ast.filter.Any(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Equals('foo'))), + ast.filter.Any(ns.bse.tag, ast.filter.Any(ns.bst.label, ast.filter.Equals('bar'))))) + self.assertEqual(self.parse('iso in ("foo", "bar") / mime = plain'), ast.filter.And( + ast.filter.Any(ns.bse.iso, ast.filter.Includes('foo', 'bar')), + ast.filter.Any(ns.bse.mime, ast.filter.Equals('plain')))) + self.assertEqual(self.parse('iso in ("foo", "bar") / mime = plain'), ast.filter.And( + ast.filter.Any(ns.bse.iso, ast.filter.Includes('foo', 'bar')), + ast.filter.Any(ns.bse.mime, ast.filter.Equals('plain')))) + self.assertEqual(self.parse('iso = 1.23 / rank < 5'), ast.filter.And( + ast.filter.Any(ns.bse.iso, ast.filter.Equals(1.23)), + ast.filter.Any(ns.bse.rank, ast.filter.LessThan(5, True)))) + # FIXME + #self.assertEqual(self.parse('time >= 12:50 / time < 13:50'), ast.filter.And( + # ast.filter.Any(ns.bse.time, ast.TimeRange(lo=datetime(1970, 1, 1, 12, 50), lo_inc=True, hi_inc=True)), + # ast.filter.Any(ns.bse.time, ast.TimeRange(hi=datetime(1970, 1, 1, 13, 50), lo_inc=True, hi_inc=False)))) + #self.assertEqual(self.parse('time >= 17.5.2001 / time < 18.4.2002'), ast.filter.And( + # ast.filter.Any(ns.bse.time, ast.Datetime(lo=datetime(2001, 5, 17, 0, 0), lo_inc=True)), + # ast.filter.Any(ns.bse.time, ast.Datetime(hi=datetime(2002, 4, 18, 0, 0))))) + # mixing expressions + #self.assertEqual(self.parse('foo / iso in "bar" / mime ~ "text/plain" / iso < 100 / time >= 17.5.2001 / time < 13:50'), ast.filter.And( + # ast.filter.Any(ns.bse.tag, ast.filter.Equals('foo')), + # ast.filter.Any(ns.bse.iso, ast.filter.Equals('bar')), + # ast.filter.Any(ns.bse.mime, ast.filter.Substring('text/plain')), + # ast.filter.Any(ns.bse.iso, ast.filter.LessThan(100)), + # ast.filter.Any(ns.bse.time, ast.Datetime(lo=datetime(2001, 5, 17, 0, 0), lo_inc=True)), + # ast.filter.Any(ns.bse.time, ast.TimeRange(hi=datetime(1970, 1, 1, 13, 50), lo_inc=True)))) + + # leading/trailing slashes + self.assertRaises(errors.ParserError, self.parse, '/ foobar') + self.assertRaises(errors.ParserError, self.parse, 'foobar /') + self.assertRaises(errors.ParserError, self.parse, 'foobar / ') + self.assertRaises(errors.ParserError, self.parse, 'foo // bar') + self.assertRaises(errors.ParserError, self.parse, 'foo / / bar') + + def test_quoting(self): + self._test("tag in ('(foo, bar)', foobar)", + ast.filter.Any(ns.bse.tag, ast.filter.Includes('(foo, bar)', 'foobar'))) + self._test('tag in ("(foo, bar)", foobar)', + ast.filter.Any(ns.bse.tag, ast.filter.Includes('(foo, bar)', 'foobar'))) + self._test('tag in ("(foo, \\"bar\\")", foobar)', + ast.filter.Any(ns.bse.tag, ast.filter.Includes('(foo, "bar")', 'foobar'))) + self._test('tag in ("(foo, bar)", "foobar")', + ast.filter.Any(ns.bse.tag, ast.filter.Includes('(foo, bar)', 'foobar'))) + self._test('tag in ("(foo, bar)", \'foobar\')', + ast.filter.Any(ns.bse.tag, ast.filter.Includes('(foo, bar)', 'foobar'))) + + # error cases + self.assertRaises(errors.ParserError, self.parse, ('tag in ("(foo, bar, foobar)')) + self.assertRaises(errors.ParserError, self.parse, ("tag in ('(foo, bar, foobar)")) + + +## main ## + +if __name__ == '__main__': + unittest.main() + +## EOF ## diff --git a/test/parsing/test_search.py b/test/parsing/test_search.py deleted file mode 100644 index 23801d0..0000000 --- a/test/parsing/test_search.py +++ /dev/null @@ -1,707 +0,0 @@ -""" - -Part of the tagit test suite. -A copy of the license is provided with the project. -Author: Matthias Baumgartner, 2022 -""" -# standard imports -import unittest -from datetime import datetime - -# external imports -from pyparsing import ParseException - -# tagit imports -from tagit.utils import errors -#from tagit.parsing.search import ast, predicates, PredicateScope # FIXME: mb/port/parsing - -# objects to test -from tagit.parsing.search import ast_from_string - - -## code ## - -class TestScope(PredicateScope): - _scope_order = ['major', 'minor', 'micro'] - _init_values = ['library'] - - -class TestParseContinuous(unittest.TestCase): - longMessage = True - - def setUp(self): - predicates.expose('mime', - TestScope('attribute', 'mime'), 'Categorical') - predicates.expose('iso', - TestScope('attribute', 'iso'), 'Continuous', 'Categorical') - predicates.expose('time', - TestScope('generic', 't_image_create_loc'), 'TimeRange', 'Datetime') - predicates.expose('tag', - TestScope('generic', 'tag'), 'Categorical') - - def _test(self, query, target): - predicate, condition = target - result = ast_from_string(query) - target = ast.AND([ast.Token(predicate, condition)]) - self.assertEqual(result, target, msg="in query '{}'".format(query)) - - def test_larger_than(self): - # larger than A (inclusive) - for editable in [ - # range - "{predicate} in [{num}:]", "{predicate} in [{num}:[", "{predicate} in [{num}:)", - "{predicate} : [{num}:]", "{predicate} : [{num}:[", "{predicate} : [{num}:)", - "{predicate} = [{num}:]", "{predicate} = [{num}:[", "{predicate} = [{num}:)", - ]: - # positive - self._test(editable.format(num=1.23, predicate='iso'), - ('iso', ast.Continuous(1.23, float('inf'), True, False))) - # negative - self._test(editable.format(num=-1.23, predicate='iso'), - ('iso', ast.Continuous(-1.23, float('inf'), True, False))) - - for editable in [ - # range - "{predicate} in [{num}-]", "{predicate} in [{num}-[", "{predicate} in [{num}-)", - "{predicate} : [{num}-]", "{predicate} : [{num}-[", "{predicate} : [{num}-)", - "{predicate} = [{num}-]", "{predicate} = [{num}-[", "{predicate} = [{num}-)", - # equation - "{predicate} >= {num}", "{num} <= {predicate}", - ]: - # positive - self._test(editable.format(num=1.23, predicate='iso'), - ('iso', ast.Continuous(1.23, float('inf'), True, False))) - # negative - self._test(editable.format(num=-1.23, predicate='iso'), - ('iso', ast.Continuous(-1.23, float('inf'), True, False))) - # date - self._test(editable.format(predicate='time', num="30.04.2012, 13:18"), - ('time', ast.Datetime(datetime(2012, 4, 30, 13, 18), datetime.max, True, False))) - - # larger than A (exclusive) - for editable in [ - # range / bracket - "{predicate} in ]{num}:]", "{predicate} in ]{num}:[", "{predicate} in ]{num}:)", - "{predicate} : ]{num}:]", "{predicate} : ]{num}:[", "{predicate} : ]{num}:)", - "{predicate} = ]{num}:]", "{predicate} = ]{num}:[", "{predicate} = ]{num}:)", - # range / parenthesis - "{predicate} in ({num}:]", "{predicate} in ({num}:[", "{predicate} in ({num}:)", - "{predicate} : ({num}:]", "{predicate} : ({num}:[", "{predicate} : ({num}:)", - "{predicate} = ({num}:]", "{predicate} = ({num}:[", "{predicate} = ({num}:)", - ]: - # positive - self._test(editable.format(num=1.23, predicate='iso'), - ('iso', ast.Continuous(1.23, float('inf'), False, False))) - # negative - self._test(editable.format(num=-1.23, predicate='iso'), - ('iso', ast.Continuous(-1.23, float('inf'), False, False))) - - for editable in [ - # range / bracket - "{predicate} in ]{num}-]", "{predicate} in ]{num}-[", "{predicate} in ]{num}-)", - "{predicate} : ]{num}-]", "{predicate} : ]{num}-[", "{predicate} : ]{num}-)", - "{predicate} = ]{num}-]", "{predicate} = ]{num}-[", "{predicate} = ]{num}-)", - # range / parenthesis - "{predicate} in ({num}-]", "{predicate} in ({num}-[", "{predicate} in ({num}-)", - "{predicate} : ({num}-]", "{predicate} : ({num}-[", "{predicate} : ({num}-)", - "{predicate} = ({num}-]", "{predicate} = ({num}-[", "{predicate} = ({num}-)", - # equation - "{predicate} > {num}", "{num} < {predicate}", - ]: - # positive - self._test(editable.format(num=1.23, predicate='iso'), - ('iso', ast.Continuous(1.23, float('inf'), False, False))) - # negative - self._test(editable.format(num=-1.23, predicate='iso'), - ('iso', ast.Continuous(-1.23, float('inf'), False, False))) - # date - self._test(editable.format(predicate='time', num="30.04.2012, 13:18"), - ('time', ast.Datetime(datetime(2012, 4, 30, 13, 19), datetime.max, True, False))) - - def test_smaller_than(self): - # smaller than B (inclusive) - for editable in [ - # range - "{predicate} in [:{num}]", "{predicate} in (:{num}]", "{predicate} in ]:{num}]", - "{predicate} : [:{num}]", "{predicate} : (:{num}]", "{predicate} : ]:{num}]", - "{predicate} = [:{num}]", "{predicate} = (:{num}]", "{predicate} = ]:{num}]", - ]: - # positives - self._test(editable.format(num=1.23, predicate='iso'), - ('iso', ast.Continuous(float('-inf'), 1.23, False, True))) - # negatives - self._test(editable.format(num=-1.23, predicate='iso'), - ('iso', ast.Continuous(float('-inf'), -1.23, False, True))) - - for editable in [ - # range - "{predicate} in [-{num}]", "{predicate} in (-{num}]", "{predicate} in ]-{num}]", - "{predicate} : [-{num}]", "{predicate} : (-{num}]", "{predicate} : ]-{num}]", - "{predicate} = [-{num}]", "{predicate} = (-{num}]", "{predicate} = ]-{num}]", - # equation - "{predicate} <={num}", "{num} >= {predicate}", - ]: - # positives - self._test(editable.format(num=1.23, predicate='iso'), - ('iso', ast.Continuous(float('-inf'), 1.23, False, True))) - # negatives - self._test(editable.format(num=-1.23, predicate='iso'), - ('iso', ast.Continuous(float('-inf'), -1.23, False, True))) - # date - self._test(editable.format(predicate='time', num="30.04.2012, 13:18"), - ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 13, 19), False, False))) - - # smaller than B (exclusive) - for editable in [ - # range / bracket - "{predicate} in [:{num}[", "{predicate} in (:{num}[", "{predicate} in ]:{num}[", - "{predicate} : [:{num}[", "{predicate} : (:{num}[", "{predicate} : ]:{num}[", - "{predicate} = [:{num}[", "{predicate} = (:{num}[", "{predicate} = ]:{num}[", - # range / parenthesis - "{predicate} in [:{num})", "{predicate} in (:{num})", "{predicate} in ]:{num})", - "{predicate} : [:{num})", "{predicate} : (:{num})", "{predicate} : ]:{num})", - "{predicate} = [:{num})", "{predicate} = (:{num})", "{predicate} = ]:{num})", - ]: - # positives - self._test(editable.format(num=1.23, predicate='iso'), - ('iso', ast.Continuous(float('-inf'), 1.23, False, False))) - # negatives - self._test(editable.format(num=-1.23, predicate='iso'), - ('iso', ast.Continuous(float('-inf'), -1.23, False, False))) - - for editable in [ - # range / bracket - "{predicate} in [-{num}[", "{predicate} in (-{num}[", "{predicate} in ]-{num}[", - "{predicate} : [-{num}[", "{predicate} : (-{num}[", "{predicate} : ]-{num}[", - "{predicate} = [-{num}[", "{predicate} = (-{num}[", "{predicate} = ]-{num}[", - # range / parenthesis - "{predicate} in [-{num})", "{predicate} in (-{num})", "{predicate} in ]-{num})", - "{predicate} : [-{num})", "{predicate} : (-{num})", "{predicate} : ]-{num})", - "{predicate} = [-{num})", "{predicate} = (-{num})", "{predicate} = ]-{num})", - # equation - "{predicate} <{num}", "{num} > {predicate}", - ]: - # positives - self._test(editable.format(num=1.23, predicate='iso'), - ('iso', ast.Continuous(float('-inf'), 1.23, False, False))) - # negatives - self._test(editable.format(num=-1.23, predicate='iso'), - ('iso', ast.Continuous(float('-inf'), -1.23, False, False))) - # date - self._test(editable.format(predicate='time', num="30.04.2012, 13:18"), - ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 13, 18), False, False))) - - def test_between(self): - # between A and B (including A, including B) - for editable in [ - # range - "{predicate} in [{numA}:{numB}]", "{predicate} : [{numA}:{numB}]", "{predicate} = [{numA}:{numB}]", - ]: - # positives - self._test(editable.format(predicate='iso', numA=1.23, numB=4.56), - ('iso', ast.Continuous(1.23, 4.56, True, True))) - # negatives - self._test(editable.format(predicate='iso', numA=-4.56, numB=-1.23), - ('iso', ast.Continuous(-4.56, -1.23, True, True))) - # mixed - self._test(editable.format(predicate='iso', numA=-1.23, numB=4.56), - ('iso', ast.Continuous(-1.23, 4.56, True, True))) - - for editable in [ - # range - "{predicate} in [{numA}-{numB}]", "{predicate} : [{numA}-{numB}]", "{predicate} = [{numA}-{numB}]", - # equation - "{numA} <= {predicate} <= {numB}" - ]: - # positives - self._test(editable.format(predicate='iso', numA=1.23, numB=4.56), - ('iso', ast.Continuous(1.23, 4.56, True, True))) - # negatives - self._test(editable.format(predicate='iso', numA=-4.56, numB=-1.23), - ('iso', ast.Continuous(-4.56, -1.23, True, True))) - # mixed - self._test(editable.format(predicate='iso', numA=-1.23, numB=4.56), - ('iso', ast.Continuous(-1.23, 4.56, True, True))) - # date - self._test(editable.format(predicate='time', numA="30.04.2012, 13:18", numB="13.6.2014, 18:27"), - ('time', ast.Datetime(datetime(2012, 4, 30, 13, 18), datetime(2014, 6, 13, 18, 28), True, False))) - - # between A and B (including A, excluding B) - for editable in [ - # range - "{predicate} in [{numA}:{numB})", "{predicate} in [{numA}:{numB}[", - "{predicate} : [{numA}:{numB})", "{predicate} : [{numA}:{numB}[", - "{predicate} = [{numA}:{numB})", "{predicate} = [{numA}:{numB}[", - ]: - # positives - self._test(editable.format(predicate='iso', numA=1.23, numB=4.56), - ('iso', ast.Continuous(1.23, 4.56, True, False))) - # negatives - self._test(editable.format(predicate='iso', numA=-4.56, numB=-1.23), - ('iso', ast.Continuous(-4.56, -1.23, True, False))) - # mixed - self._test(editable.format(predicate='iso', numA=-1.23, numB=4.56), - ('iso', ast.Continuous(-1.23, 4.56, True, False))) - - for editable in [ - # range - "{predicate} in [{numA}-{numB})", "{predicate} in [{numA}-{numB}[", - "{predicate} : [{numA}-{numB})", "{predicate} : [{numA}-{numB}[", - "{predicate} = [{numA}-{numB})", "{predicate} = [{numA}-{numB}[", - # equation - "{numA} <= {predicate} < {numB}", - ]: - # positives - self._test(editable.format(predicate='iso', numA=1.23, numB=4.56), - ('iso', ast.Continuous(1.23, 4.56, True, False))) - # negatives - self._test(editable.format(predicate='iso', numA=-4.56, numB=-1.23), - ('iso', ast.Continuous(-4.56, -1.23, True, False))) - # mixed - self._test(editable.format(predicate='iso', numA=-1.23, numB=4.56), - ('iso', ast.Continuous(-1.23, 4.56, True, False))) - # date - self._test(editable.format(predicate='time', numA="30.04.2012, 13:18", numB="13.6.2014, 18:27"), - ('time', ast.Datetime(datetime(2012, 4, 30, 13, 18), datetime(2014, 6, 13, 18, 27), True, False))) - - # between A and B (excluding A, including B) - for editable in [ - # range - "{predicate} in ({numA}:{numB}]", "{predicate} in ]{numA}:{numB}]", - "{predicate} : ({numA}:{numB}]", "{predicate} : ]{numA}:{numB}]", - "{predicate} = ({numA}:{numB}]", "{predicate} = ]{numA}:{numB}]", - ]: - # positives - self._test(editable.format(predicate='iso', numA=1.23, numB=4.56), - ('iso', ast.Continuous(1.23, 4.56, False, True))) - # negatives - self._test(editable.format(predicate='iso', numA=-4.56, numB=-1.23), - ('iso', ast.Continuous(-4.56, -1.23, False, True))) - # mixed - self._test(editable.format(predicate='iso', numA=-1.23, numB=4.56), - ('iso', ast.Continuous(-1.23, 4.56, False, True))) - - for editable in [ - # range - "{predicate} in ({numA}-{numB}]", "{predicate} in ]{numA}-{numB}]", - "{predicate} : ({numA}-{numB}]", "{predicate} : ]{numA}-{numB}]", - "{predicate} = ({numA}-{numB}]", "{predicate} = ]{numA}-{numB}]", - # equation - "{numA} < {predicate} <= {numB}", - ]: - # positives - self._test(editable.format(predicate='iso', numA=1.23, numB=4.56), - ('iso', ast.Continuous(1.23, 4.56, False, True))) - # negatives - self._test(editable.format(predicate='iso', numA=-4.56, numB=-1.23), - ('iso', ast.Continuous(-4.56, -1.23, False, True))) - # mixed - self._test(editable.format(predicate='iso', numA=-1.23, numB=4.56), - ('iso', ast.Continuous(-1.23, 4.56, False, True))) - # date - self._test(editable.format(predicate='time', numA="30.04.2012, 13:18", numB="13.6.2014, 18:27"), - ('time', ast.Datetime(datetime(2012, 4, 30, 13, 19), datetime(2014, 6, 13, 18, 28), True, False))) - - # between A and B (excluding A, excluding B) - for editable in [ - "{predicate} in ({numA}:{numB})", "{predicate} in ]{numA}:{numB}[", - "{predicate} : ({numA}:{numB})", "{predicate} : ]{numA}:{numB}[", - "{predicate} = ({numA}:{numB})", "{predicate} = ]{numA}:{numB}[", - ]: - # positives - self._test(editable.format(predicate='iso', numA=1.23, numB=4.56), - ('iso', ast.Continuous(1.23, 4.56, False, False))) - # negatives - self._test(editable.format(predicate='iso', numA=-4.56, numB=-1.23), - ('iso', ast.Continuous(-4.56, -1.23, False, False))) - # mixed - self._test(editable.format(predicate='iso', numA=-1.23, numB=4.56), - ('iso', ast.Continuous(-1.23, 4.56, False, False))) - - for editable in [ - "{predicate} in ({numA}-{numB})", "{predicate} in ]{numA}-{numB}[", - "{predicate} : ({numA}-{numB})", "{predicate} : ]{numA}-{numB}[", - "{predicate} = ({numA}-{numB})", "{predicate} = ]{numA}-{numB}[", - # equation - "{numA} < {predicate} < {numB}", - ]: - # positives - self._test(editable.format(predicate='iso', numA=1.23, numB=4.56), - ('iso', ast.Continuous(1.23, 4.56, False, False))) - # negatives - self._test(editable.format(predicate='iso', numA=-4.56, numB=-1.23), - ('iso', ast.Continuous(-4.56, -1.23, False, False))) - # mixed - self._test(editable.format(predicate='iso', numA=-1.23, numB=4.56), - ('iso', ast.Continuous(-1.23, 4.56, False, False))) - # date - self._test(editable.format(predicate='time', numA="30.04.2012, 13:18", numB="13.6.2014, 18:27"), - ('time', ast.Datetime(datetime(2012, 4, 30, 13, 19), datetime(2014, 6, 13, 18, 27), True, False))) - - def test_equal(self): - # equal to A - for editable in [ - # range - "{predicate} in [{num}:{num}]", "{predicate} : [{num}:{num}]", "{predicate} = [{num}:{num}]", - ]: - # positives - self._test(editable.format(predicate='iso', num=1.23), - ('iso', ast.Continuous(1.23, 1.23, True, True))) - # negatives - self._test(editable.format(predicate='iso', num=-1.23), - ('iso', ast.Continuous(-1.23, -1.23, True, True))) - - for editable in [ - # range - "{predicate} in [{num}-{num}]", "{predicate} : [{num}-{num}]", "{predicate} = [{num}-{num}]", - # equation - "{predicate} = {num}", "{num} = {predicate}", - ]: - # positives - self._test(editable.format(predicate='iso', num=1.23), - ('iso', ast.Continuous(1.23, 1.23, True, True))) - # negatives - self._test(editable.format(predicate='iso', num=-1.23), - ('iso', ast.Continuous(-1.23, -1.23, True, True))) - # date - self._test(editable.format(predicate='time', num="30.04.2012, 13:18"), - ('time', ast.Datetime(datetime(2012, 4, 30, 13, 18), datetime(2012, 4, 30, 13, 19), True, False))) - - def test_dates(self): - self._test("{predicate} < {num}".format(predicate='time', num="2012"), - ('time', ast.Datetime(datetime.min, datetime(2012, 1, 1), False, False))) - self._test("{predicate} < {num}".format(predicate='time', num="2012.04"), - ('time', ast.Datetime(datetime.min, datetime(2012, 4, 1), False, False))) - self._test("{predicate} < {num}".format(predicate='time', num="2012.04.30"), - ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30), False, False))) - self._test("{predicate} < {num}".format(predicate='time', num="2012.04.30, 3 pm"), - ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 15), False, False))) - self._test("{predicate} < {num}".format(predicate='time', num="2012.04.30, 15:34"), - ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 15, 34), False, False))) - self._test("{predicate} < {num}".format(predicate='time', num="2012.04.30, 15:34:12"), - ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 15, 34, 12), False, False))) - self._test("{predicate} < {num}".format(predicate='time', num="2012.04.30, 15:34:12.98"), - ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 15, 34, 12, 980000), False, False))) - - self._test("{predicate} <= {num}".format(predicate='time', num="2012"), - ('time', ast.Datetime(datetime.min, datetime(2013, 1, 1), False, False))) - self._test("{predicate} <= {num}".format(predicate='time', num="2012.04"), - ('time', ast.Datetime(datetime.min, datetime(2012, 5, 1), False, False))) - self._test("{predicate} <= {num}".format(predicate='time', num="2012.04.30"), - ('time', ast.Datetime(datetime.min, datetime(2012, 5, 1), False, False))) - self._test("{predicate} <= {num}".format(predicate='time', num="2012.04.30, 3 pm"), - ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 16), False, False))) - self._test("{predicate} <= {num}".format(predicate='time', num="2012.04.30, 15:34"), - ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 15, 35), False, False))) - self._test("{predicate} <= {num}".format(predicate='time', num="2012.04.30, 15:34:12"), - ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 15, 34, 13), False, False))) - self._test("{predicate} <= {num}".format(predicate='time', num="2012.04.30, 15:34:12.98"), - ('time', ast.Datetime(datetime.min, datetime(2012, 4, 30, 15, 34, 12, 980001), False, False))) - - def test_timerange(self): - self._test("{predicate} < {num}".format(predicate='time', num="15:34"), - ('time', ast.TimeRange(datetime.utcfromtimestamp(0.0), datetime(1970, 1, 1, 15, 34), True, False))) - self._test("{predicate} <= {num}".format(predicate='time', num="15:34"), - ('time', ast.TimeRange(datetime.utcfromtimestamp(0.0), datetime(1970, 1, 1, 15, 35), True, False))) - self._test("{predicate} = {num}".format(predicate='time', num="15:34"), - ('time', ast.TimeRange(datetime(1970, 1, 1, 15, 34), datetime(1970, 1, 1, 15, 35), True, False))) - self._test("{predicate} > {num}".format(predicate='time', num="15:34"), - ('time', ast.TimeRange(datetime(1970, 1, 1, 15, 35), datetime(1970, 1, 2), True, True))) - self._test("{predicate} >= {num}".format(predicate='time', num="15:34"), - ('time', ast.TimeRange(datetime(1970, 1, 1, 15, 34), datetime(1970, 1, 2), True, True))) - - self._test("{numA} <= {predicate} <= {numB}".format(predicate='time', numA="12:34", numB="15:28"), - ('time', ast.TimeRange(datetime(1970, 1, 1, 12, 34), datetime(1970, 1, 1, 15, 29), True, False))) - self._test("{numA} <= {predicate} < {numB}".format(predicate='time', numA="12:34", numB="15:28"), - ('time', ast.TimeRange(datetime(1970, 1, 1, 12, 34), datetime(1970, 1, 1, 15, 28), True, False))) - self._test("{numA} < {predicate} <= {numB}".format(predicate='time', numA="12:34", numB="15:28"), - ('time', ast.TimeRange(datetime(1970, 1, 1, 12, 35), datetime(1970, 1, 1, 15, 29), True, False))) - self._test("{numA} < {predicate} < {numB}".format(predicate='time', numA="12:34", numB="15:28"), - ('time', ast.TimeRange(datetime(1970, 1, 1, 12, 35), datetime(1970, 1, 1, 15, 28), True, False))) - - def test_special(self): - # special cases: explicit plus sign - self._test("{predicate} in [+1.23-+4.56]".format(predicate='iso'), - ('iso', ast.Continuous(1.23, 4.56, True, True))) - self._test("{predicate} in [-+4.56]".format(predicate='iso'), - ('iso', ast.Continuous(float('-inf'), 4.56, False, True))) - - def test_errors(self): - # parse errors - for editable in [ - # equal with exclusive - "{predicate} in ({num}:{num})", "{predicate} in ({num}-{num})", - "{predicate} in ({num}:{num}[", "{predicate} in ({num}-{num}[", - "{predicate} in ]{num}:{num})", "{predicate} in ]{num}-{num})", - "{predicate} in ]{num}:{num}[", "{predicate} in ]{num}-{num}[", - # invalid parentesis - "{predicate} in ){num}:{num}(", - # misc errors - # FIXME: Currently all special characters are allowed as categorical value. - # If this changes, don't forget to enable the tests below. - #"{predicate} in [{num}{num}]", - #"{predicate} [{num}:{num}:{num}]", - #"{predicate} = ({num})", - #"{predicate} = {num})", - ]: - self.assertRaises(errors.ParserError, ast_from_string, - editable.format(predicate='iso', num=1.23)) - - for editable in [ - "{predicate} in [{numA}:{numB}]", "{predicate} : [{numA}:{numB}]", "{predicate} = [{numA}:{numB}]", - "{predicate} in ]{numA}:{numB}]", "{predicate} : ]{numA}:{numB}]", "{predicate} = ]{numA}:{numB}]", - "{predicate} in [{numA}:{numB}[", "{predicate} : [{numA}:{numB}[", "{predicate} = [{numA}:{numB}[", - "{predicate} in ({numA}:{numB}]", "{predicate} : ({numA}:{numB}]", "{predicate} = ({numA}:{numB}]", - "{predicate} in [{numA}:{numB})", "{predicate} : [{numA}:{numB})", "{predicate} = [{numA}:{numB})", - "{predicate} in ]{numA}:{numB}[", "{predicate} : ]{numA}:{numB}[", "{predicate} = ]{numA}:{numB}[", - "{predicate} in ]{numA}:{numB})", "{predicate} : ]{numA}:{numB})", "{predicate} = ]{numA}:{numB})", - "{predicate} in ({numA}:{numB}[", "{predicate} : ({numA}:{numB}[", "{predicate} = ({numA}:{numB}[", - "{predicate} in ({numA}:{numB})", "{predicate} : ({numA}:{numB})", "{predicate} = ({numA}:{numB})", - "{numA} < {predicate} < {numB}", - "{numA} <= {predicate} < {numB}", - "{numA} < {predicate} <= {numB}", - ]: - self.assertRaises(errors.ParserError, ast_from_string, - editable.format(predicate='iso', numA=4.56, numB=1.23)) - self.assertRaises(errors.ParserError, ast_from_string, - editable.format(predicate='time', numA="17:35", numB="10:55")) - self.assertRaises(errors.ParserError, ast_from_string, - editable.format(predicate='time', numA="18.12.2035", numB="5.7.1999")) - - # special cases: empty range with boundary - self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, - "{predicate} in [:]".format(predicate='iso')) - self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, - "{predicate} in (:[".format(predicate='iso')) - self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, - "{predicate} in ]:)".format(predicate='iso')) - self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, - "{predicate} in ".format(predicate='iso')) - # misc - self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, - "{predicate} in [{num}{num}]".format(predicate='iso', num=1.23)) - self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, - "{predicate} [{num}:{num}:{num}]".format(predicate='iso', num=1.23)) - self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, - "{predicate} = ({num})".format(predicate='iso', num=1.23)) - self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, - "{predicate} = ({num}".format(predicate='iso', num=1.23), dict(parseAll=True)) - self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, - "{predicate} = {num})".format(predicate='iso', num=1.23), dict(parseAll=True)) - # range errors - self.assertRaises(errors.ParserError, ast_from_string, "100 >= iso < 200") - self.assertRaises(errors.ParserError, ast_from_string, "100 > iso < 200") - self.assertRaises(errors.ParserError, ast_from_string, "100 > iso <= 200") - self.assertRaises(errors.ParserError, ast_from_string, "100 >= iso <= 200") - self.assertRaises(errors.ParserError, ast_from_string, "100 = iso = 200") - # time/date mixture errors - self.assertRaises(errors.ParserError, ast_from_string, "12:45 < time < 17.5.2004") - self.assertRaises(errors.ParserError, ast_from_string, "17.5.2004 < time < 12:45") - # date/int mixture errors - self.assertRaises(errors.ParserError, ast_from_string, "17.5.2004 < time < 1245") - # 1245 is interpreted as the year - #self.assertRaises(errors.ParserError, ast_from_string, "1245 < time < 17.5.2004") - # time/int mixture errors - self.assertRaises(errors.ParserError, ast_from_string, "17:12 < time < 1245") - self.assertRaises(errors.ParserError, ast_from_string, "1712 < time < 12:45") - - # empty query - self.assertRaises(ParseException, ast_from_string.CONTINUOUS.parseString, "") - - -class TestParseSearch(unittest.TestCase): - def setUp(self): - predicates.expose('mime', - TestScope('attribute', 'mime'), 'Categorical') - predicates.expose('rank', - TestScope('attribute', 'rank'), 'Continuous') - predicates.expose('iso', - TestScope('attribute', 'iso'), 'Continuous', 'Categorical') - predicates.expose('time', - TestScope('generic', 't_image_create_loc'), 'TimeRange', 'Datetime') - predicates.expose('tag', - TestScope('generic', 'tag'), 'Categorical') - - def test_parse_existence(self): - self.assertEqual(ast_from_string("has mime"), - ast.AND([ast.Token('mime', ast.Existence())])) - self.assertEqual(ast_from_string("has no mime"), - ast.AND([ast.Token('mime', ast.Inexistence())])) - self.assertEqual(ast_from_string("has not mime"), - ast.AND([ast.Token('mime', ast.Inexistence())])) - - def test_parse_categorical(self): - # positive - self.assertEqual(ast_from_string("iso in 100, 200, 500"), - ast.AND([ast.Token('iso', ast.SetInclude(['100', '200', '500']))])) - self.assertEqual(ast_from_string("iso in (100, 200)"), - ast.AND([ast.Token('iso', ast.SetInclude(['100', '200']))])) - self.assertEqual(ast_from_string("iso = (100, 200)"), - ast.AND([ast.Token('iso', ast.SetInclude(['100', '200']))])) - # FIXME! - #self.assertEqual(ast_from_string("iso = 100, 200"), - # ast.AND([ast.Token('iso', ast.SetInclude(['100', '200']))])) - self.assertEqual(ast_from_string("iso : (100, 200)"), - ast.AND([ast.Token('iso', ast.SetInclude(['100', '200']))])) - self.assertEqual(ast_from_string("iso : 100, 200"), - ast.AND([ast.Token('iso', ast.SetInclude(['100', '200']))])) - self.assertEqual(ast_from_string("iso:(100,200)"), - ast.AND([ast.Token('iso', ast.SetInclude(['100', '200']))])) - self.assertEqual(ast_from_string("iso in (100,200)"), - ast.AND([ast.Token('iso', ast.SetInclude(['100', '200']))])) - self.assertEqual(ast_from_string("iso in 100,200"), - ast.AND([ast.Token('iso', ast.SetInclude(['100', '200']))])) - self.assertEqual(ast_from_string("iso ~ (100,200)"), - ast.AND([ast.Token('iso', ast.SetInclude(['100', '200'], approximate=True))])) - self.assertEqual(ast_from_string("iso ~ 100,200"), - ast.AND([ast.Token('iso', ast.SetInclude(['100', '200'], approximate=True))])) - - # negative - self.assertEqual(ast_from_string("iso not in 100,200"), - ast.AND([ast.Token('iso', ast.SetExclude(['100', '200']))])) - self.assertEqual(ast_from_string("iso not in (100, 200)"), - ast.AND([ast.Token('iso', ast.SetExclude(['100', '200']))])) - self.assertEqual(ast_from_string("iso != 100,200"), - ast.AND([ast.Token('iso', ast.SetExclude(['100', '200']))])) - self.assertEqual(ast_from_string("iso != (100, 200)"), - ast.AND([ast.Token('iso', ast.SetExclude(['100', '200']))])) - self.assertEqual(ast_from_string("iso !~ 100,200"), - ast.AND([ast.Token('iso', ast.SetExclude(['100', '200'], approximate=True))])) - self.assertEqual(ast_from_string("iso !~ (100, 200)"), - ast.AND([ast.Token('iso', ast.SetExclude(['100', '200'], approximate=True))])) - - # one value - self.assertEqual(ast_from_string("mime : text"), - ast.AND([ast.Token('mime', ast.SetInclude(['text']))])) - self.assertEqual(ast_from_string("mime in text"), - ast.AND([ast.Token('mime', ast.SetInclude(['text']))])) - self.assertEqual(ast_from_string("mime = text"), - ast.AND([ast.Token('mime', ast.SetInclude(['text']))])) - self.assertEqual(ast_from_string("mime ~ text"), - ast.AND([ast.Token('mime', ast.SetInclude(['text'], approximate=True))])) - self.assertEqual(ast_from_string("mime != text"), - ast.AND([ast.Token('mime', ast.SetExclude(['text']))])) - self.assertEqual(ast_from_string("mime not in text"), - ast.AND([ast.Token('mime', ast.SetExclude(['text']))])) - self.assertEqual(ast_from_string("mime !~ text"), - ast.AND([ast.Token('mime', ast.SetExclude(['text'], approximate=True))])) - - # expressions with slash and comma - self.assertEqual(ast_from_string('mime : "text"'), - ast.AND([ast.Token('mime', ast.SetInclude(['text']))])) - self.assertEqual(ast_from_string('mime : "text", "plain"'), - ast.AND([ast.Token('mime', ast.SetInclude(['text', 'plain']))])) - self.assertEqual(ast_from_string('mime : "text, plain"'), - ast.AND([ast.Token('mime', ast.SetInclude(['text, plain']))])) - self.assertEqual(ast_from_string('mime ~ "text/plain"'), - ast.AND([ast.Token('mime', ast.SetInclude(['text/plain'], approximate=True))])) - self.assertEqual(ast_from_string('mime = ("text/plain", "image/jpeg")'), - ast.AND([ast.Token('mime', ast.SetInclude(['text/plain', 'image/jpeg']))])) - - def test_parse_tag(self): - - # only tag: tag, tags, (tag), (tags) - self.assertEqual(ast_from_string("foo"), - ast.AND([ast.Token('tag', ast.SetInclude(['foo']))])) - self.assertEqual(ast_from_string("(foo)"), - ast.AND([ast.Token('tag', ast.SetInclude(['foo']))])) - self.assertEqual(ast_from_string("foo, bar"), - ast.AND([ast.Token('tag', ast.SetInclude(['foo', 'bar']))])) - self.assertEqual(ast_from_string("foo,bar"), - ast.AND([ast.Token('tag', ast.SetInclude(['foo', 'bar']))])) - self.assertEqual(ast_from_string("(foo, bar,foobar)"), - ast.AND([ast.Token('tag', ast.SetInclude(['foo', 'bar', 'foobar']))])) - - # op and tag: !tag, ~tag, !~tag - self.assertEqual(ast_from_string("~foo"), - ast.AND([ast.Token('tag', ast.SetInclude(['foo'], approximate=True))])) - self.assertEqual(ast_from_string("~ foo"), - ast.AND([ast.Token('tag', ast.SetInclude(['foo'], approximate=True))])) - self.assertEqual(ast_from_string("!foo"), - ast.AND([ast.Token('tag', ast.SetExclude(['foo']))])) - self.assertEqual(ast_from_string("! foo"), - ast.AND([ast.Token('tag', ast.SetExclude(['foo']))])) - self.assertEqual(ast_from_string("!~foo"), - ast.AND([ast.Token('tag', ast.SetExclude(['foo'], approximate=True))])) - self.assertEqual(ast_from_string("!~ foo"), - ast.AND([ast.Token('tag', ast.SetExclude(['foo'], approximate=True))])) - - # op and list: ! (tags), ~tags, ... - self.assertEqual(ast_from_string("~ foo, bar"), - ast.AND([ast.Token('tag', ast.SetInclude(['foo', 'bar'], approximate=True))])) - self.assertEqual(ast_from_string("~foo, bar"), - ast.AND([ast.Token('tag', ast.SetInclude(['foo', 'bar'], approximate=True))])) - self.assertEqual(ast_from_string("~ (foo, bar)"), - ast.AND([ast.Token('tag', ast.SetInclude(['foo', 'bar'], approximate=True))])) - self.assertEqual(ast_from_string("! foo, bar"), - ast.AND([ast.Token('tag', ast.SetExclude(['foo', 'bar']))])) - self.assertEqual(ast_from_string("! (foo, bar)"), - ast.AND([ast.Token('tag', ast.SetExclude(['foo', 'bar']))])) - self.assertEqual(ast_from_string("! (foo,bar)"), - ast.AND([ast.Token('tag', ast.SetExclude(['foo', 'bar']))])) - self.assertEqual(ast_from_string("!~ foo, bar"), - ast.AND([ast.Token('tag', ast.SetExclude(['foo', 'bar'], approximate=True))])) - self.assertEqual(ast_from_string("!~ (foo, bar)"), - ast.AND([ast.Token('tag', ast.SetExclude(['foo', 'bar'], approximate=True))])) - self.assertEqual(ast_from_string("!~(foo,bar)"), - ast.AND([ast.Token('tag', ast.SetExclude(['foo', 'bar'], approximate=True))])) - - def test_parse_query(self): - # simple query - self.assertEqual(ast_from_string('foo / bar'), ast.AND([ - ast.Token('tag', ast.SetInclude('foo')), - ast.Token('tag', ast.SetInclude('bar'))])) - self.assertEqual(ast_from_string('iso in ("foo", "bar") / mime = plain'), ast.AND([ - ast.Token('iso', ast.SetInclude('foo', 'bar')), - ast.Token('mime', ast.SetInclude('plain'))])) - self.assertEqual(ast_from_string('iso in ("foo", "bar") / mime = plain'), ast.AND([ - ast.Token('iso', ast.SetInclude('foo', 'bar')), - ast.Token('mime', ast.SetInclude('plain'))])) - self.assertEqual(ast_from_string('iso = 1.23 / rank < 5'), ast.AND([ - ast.Token('iso', ast.Continuous(1.23, 1.23, True, True)), - ast.Token('rank', ast.Continuous(hi=5))])) - self.assertEqual(ast_from_string('time >= 12:50 / time < 13:50'), ast.AND([ - ast.Token('time', ast.TimeRange(lo=datetime(1970, 1, 1, 12, 50), lo_inc=True, hi_inc=True)), - ast.Token('time', ast.TimeRange(hi=datetime(1970, 1, 1, 13, 50), lo_inc=True, hi_inc=False))])) - self.assertEqual(ast_from_string('time >= 17.5.2001 / time < 18.4.2002'), ast.AND([ - ast.Token('time', ast.Datetime(lo=datetime(2001, 5, 17, 0, 0), lo_inc=True)), - ast.Token('time', ast.Datetime(hi=datetime(2002, 4, 18, 0, 0)))])) - # mixing expressions - self.assertEqual(ast_from_string('foo / iso in "bar" / mime ~ "text/plain" / iso < 100 / time >= 17.5.2001 / time < 13:50'), ast.AND([ - ast.Token('tag', ast.SetInclude('foo')), - ast.Token('iso', ast.SetInclude('bar')), - ast.Token('mime', ast.SetInclude('text/plain', approximate=True)), - ast.Token('iso', ast.Continuous(hi=100)), - ast.Token('time', ast.Datetime(lo=datetime(2001, 5, 17, 0, 0), lo_inc=True)), - ast.Token('time', ast.TimeRange(hi=datetime(1970, 1, 1, 13, 50), lo_inc=True))])) - - # leading/trailing slashes - self.assertRaises(errors.ParserError, ast_from_string, '/ foobar') - self.assertRaises(errors.ParserError, ast_from_string, 'foobar /') - self.assertRaises(errors.ParserError, ast_from_string, 'foobar / ') - self.assertRaises(errors.ParserError, ast_from_string, 'foo // bar') - self.assertRaises(errors.ParserError, ast_from_string, 'foo / / bar') - - def test_quoting(self): - self.assertEqual(ast_from_string("tag in ('(foo, bar)', foobar)"), - ast.AND([ast.Token('tag', ast.SetInclude(['(foo, bar)', 'foobar']))])) - self.assertEqual(ast_from_string('tag in ("(foo, bar)", foobar)'), - ast.AND([ast.Token('tag', ast.SetInclude(['(foo, bar)', 'foobar']))])) - self.assertEqual(ast_from_string('tag in ("(foo, \\"bar\\")", foobar)'), - ast.AND([ast.Token('tag', ast.SetInclude(['(foo, "bar")', 'foobar']))])) - self.assertEqual(ast_from_string('tag in ("(foo, bar)", "foobar")'), - ast.AND([ast.Token('tag', ast.SetInclude(['(foo, bar)', 'foobar']))])) - self.assertEqual(ast_from_string('tag in ("(foo, bar)", \'foobar\')'), - ast.AND([ast.Token('tag', ast.SetInclude(['(foo, bar)', 'foobar']))])) - - # error cases - self.assertRaises(errors.ParserError, ast_from_string, ('tag in ("(foo, bar, foobar)')) - self.assertRaises(errors.ParserError, ast_from_string, ("tag in ('(foo, bar, foobar)")) - - -## main ## - -if __name__ == '__main__': - unittest.main() - -## EOF ## -- cgit v1.2.3