From 3837a849a3cf853b045ea4c0f2c18a93b4b74347 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Tue, 2 Jun 2020 19:04:53 +0200 Subject: [PATCH 01/26] Run mypy on travis --- .travis.yml | 1 + Makefile | 3 +++ 2 files changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index c5fb91b..20d628d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,4 +8,5 @@ install: - pip install xtermcolor script: + - make mypy - make test diff --git a/Makefile b/Makefile index 4741ed2..c11c765 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,9 @@ relational_gui/rel_edit.py: relational_gui/resources.py: pyrcc5 relational_gui/resources.qrc > relational_gui/resources.py +.PHONY: mypy +mypy: + mypy relational .PHONY: test test: From 4ef4d679ac6aa10e42e47e00642a692e2c641e60 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Tue, 2 Jun 2020 22:53:55 +0200 Subject: [PATCH 02/26] Remove object inheritance --- relational/relation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/relational/relation.py b/relational/relation.py index a3135e9..1f564fd 100644 --- a/relational/relation.py +++ b/relational/relation.py @@ -27,7 +27,7 @@ from typing import List, Union, Set from relational.rtypes import * -class Relation (object): +class Relation: ''' This object defines a relation (as a group of consistent tuples) and operations. @@ -53,9 +53,9 @@ class Relation (object): ''' __hash__ = None # type: None - def __init__(self, filename : str = '') -> None: + def __init__(self, filename: str = '') -> None: self._readonly = False - self.content = set() # type: Set[tuple] + self.content: Set[tuple] = set() if len(filename) == 0: # Empty relation self.header = Header([]) From 6a33e8ee0816896b64ef492874f6da6b236c84ae Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Wed, 3 Jun 2020 07:01:52 +0200 Subject: [PATCH 03/26] Improve type annotations --- relational/relation.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/relational/relation.py b/relational/relation.py index 1f564fd..346730d 100644 --- a/relational/relation.py +++ b/relational/relation.py @@ -22,7 +22,8 @@ import csv from itertools import chain, repeat from collections import deque -from typing import List, Union, Set +from typing import * +from pathlib import Path from relational.rtypes import * @@ -53,11 +54,11 @@ class Relation: ''' __hash__ = None # type: None - def __init__(self, filename: str = '') -> None: + def __init__(self, filename: Optional[Union[str, Path]] = None) -> None: self._readonly = False self.content: Set[tuple] = set() - if len(filename) == 0: # Empty relation + if filename is None: # Empty relation self.header = Header([]) return with open(filename) as fp: @@ -73,7 +74,7 @@ class Relation: self._readonly = True copy._readonly = True - def _make_writable(self, copy_content : bool = True) -> None: + def _make_writable(self, copy_content: bool = True) -> None: '''If this relation is marked as readonly, this method will copy the content to make it writable too @@ -92,7 +93,7 @@ class Relation: def __contains__(self, key): return key in self.content - def save(self, filename: str) -> None: + def save(self, filename: Union[Path, str]) -> None: ''' Saves the relation in a file. Will save using the csv format as defined in RFC4180. @@ -169,7 +170,7 @@ class Relation: newt.content.add(i + j) return newt - def projection(self, * attributes) -> 'Relation': + def projection(self, *attributes) -> 'Relation': ''' Can be called in two different ways: a.projection('field1','field2') @@ -200,7 +201,7 @@ class Relation: newt.content.add(tuple(row)) return newt - def rename(self, params: 'Relation') -> 'Relation': + def rename(self, params: Dict[str, str]) -> 'Relation': ''' Takes a dictionary. @@ -505,7 +506,7 @@ class Header(tuple): def __repr__(self): return "Header(%s)" % super(Header, self).__repr__() - def rename(self, params) -> 'Header': + def rename(self, params: Dict[str, str]) -> 'Header': '''Returns a new header, with renamed fields. params is a dictionary of {old:new} names @@ -525,15 +526,15 @@ class Header(tuple): '''Returns how many attributes this header has in common with a given one''' return len(set(self).intersection(set(other))) - def union(self, other) -> set: + def union(self, other: 'Header') -> Set[str]: '''Returns the union of the sets of attributes with another header.''' return set(self).union(set(other)) - def intersection(self, other) -> set: + def intersection(self, other: 'Header') -> Set[str]: '''Returns the set of common attributes with another header.''' return set(self).intersection(set(other)) - def getAttributesId(self, param) -> List[int]: + def getAttributesId(self, param: Iterable[str]) -> List[int]: '''Returns a list with numeric index corresponding to field's name''' try: return [self.index(i) for i in param] From 15f5de6958ca46bbc8e2863db4ce8788690aa7c0 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Sun, 14 Jun 2020 22:28:09 +0200 Subject: [PATCH 04/26] Raise exception, will fix it later --- relational/optimizations.py | 35 ----------------------------------- relational/optimizer.py | 1 + 2 files changed, 1 insertion(+), 35 deletions(-) diff --git a/relational/optimizations.py b/relational/optimizations.py index 9e05bef..51819a7 100644 --- a/relational/optimizations.py +++ b/relational/optimizations.py @@ -64,41 +64,6 @@ def find_duplicates(node, dups=None): dups[str(node)] = node - -def replace_leaves(node, context): - ''' - Node is a parsed tree - context is a dictionary containing - parsed trees as values. - - If a name appearing in node appears - also in context, the parse tree is - modified to replace the node with the - subtree found in context. - ''' - if node.kind == parser.UNARY: - replace_leaves(node.child, context) - elif node.kind == parser.BINARY: - replace_leaves(node.left, context) - replace_leaves(node.right, context) - elif node.name in context: - replace_node(node, context[node.name]) - - -def replace_node(replace, replacement): - '''This function replaces "replace" node with the node "with", - the father of the node will now point to the with node''' - replace.name = replacement.name - replace.kind = replacement.kind - - if replace.kind == parser.UNARY: - replace.child = replacement.child - replace.prop = replacement.prop - elif replace.kind == parser.BINARY: - replace.right = replacement.right - replace.left = replacement.left - - def duplicated_select(n: parser.Node) -> Tuple[parser.Node, int]: '''This function locates and deletes things like σ a ( σ a(C)) and the ones like σ a ( σ b(C)) diff --git a/relational/optimizer.py b/relational/optimizer.py index 7a4c6d7..8976f31 100644 --- a/relational/optimizer.py +++ b/relational/optimizer.py @@ -36,6 +36,7 @@ def optimize_program(code, rels: Dict[str, Relation]): Optimize an entire program, composed by multiple expressions and assignments. ''' + raise NotImplementedError() lines = code.split('\n') context = {} From aa6568bac288fb5bf7a2c9c016ddab562cd6d5b4 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Sun, 14 Jun 2020 22:30:39 +0200 Subject: [PATCH 05/26] Check class too when matching --- relational/optimizations.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/relational/optimizations.py b/relational/optimizations.py index 51819a7..d9f15e2 100644 --- a/relational/optimizations.py +++ b/relational/optimizations.py @@ -72,7 +72,7 @@ def duplicated_select(n: parser.Node) -> Tuple[parser.Node, int]: in and ''' changes = 0 - while n.name == SELECTION and n.child.name == SELECTION: + while isinstance(n, parser.UNARY) and n.name == SELECTION and isinstance(n.child, parser.UNARY) and n.child.name == SELECTION: changes += 1 prop = n.prop @@ -147,7 +147,7 @@ def down_to_unions_subtractions_intersections(n: parser.Node) -> Tuple[parser.No ''' changes = 0 _o = (UNION, DIFFERENCE, INTERSECTION) - if n.name == SELECTION and n.child.name in _o: + if isinstance(n, parser.UNARY) and n.name == SELECTION and n.child.name in _o: l = parser.Unary(SELECTION, n.prop, n.child.left) r = parser.Unary(SELECTION, n.prop, n.child.right) @@ -170,7 +170,7 @@ def duplicated_projection(n: parser.Node) -> Tuple[parser.Node, int]: def selection_inside_projection(n: parser.Node) -> Tuple[parser.Node, int]: '''This function locates things like σ j (π k(R)) and converts them into π k(σ j (R))''' - if n.name == SELECTION and n.child.name == PROJECTION: + if isinstance(n, parser.UNARY) and n.name == SELECTION and n.child.name == PROJECTION: child = parser.Unary( SELECTION, n.prop, @@ -338,7 +338,7 @@ def swap_rename_select(n: parser.Node) -> int: Renaming the attributes used in the selection, so the operation is still valid.''' - if n.name == SELECTION and n.child.name == RENAME: + if isinstance(n, parser.UNARY) and n.name == SELECTION and n.child.name == RENAME: # This is an inverse mapping for the rename renames = {v: k for k, v in n.child.get_rename_prop().items()} @@ -438,7 +438,7 @@ def selection_and_product(n: parser.Node, rels: Dict[str, Relation]) -> parser.N σ l (σ j (R) * σ i (Q)). Where j contains only attributes belonging to R, i contains attributes belonging to Q and l contains attributes belonging to both''' - if n.name == SELECTION and n.child.name in (PRODUCT, JOIN): + if isinstance(n, parser.UNARY) and n.name == SELECTION and n.child.name in (PRODUCT, JOIN): l_attr = n.child.left.result_format(rels) r_attr = n.child.right.result_format(rels) From 4ab01c9e4783cc3b1b9af93f29ff72e5b9d97a8e Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Sun, 14 Jun 2020 22:34:27 +0200 Subject: [PATCH 06/26] Not a constant --- relational/optimizations.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/relational/optimizations.py b/relational/optimizations.py index d9f15e2..7a9ee62 100644 --- a/relational/optimizations.py +++ b/relational/optimizations.py @@ -72,7 +72,7 @@ def duplicated_select(n: parser.Node) -> Tuple[parser.Node, int]: in and ''' changes = 0 - while isinstance(n, parser.UNARY) and n.name == SELECTION and isinstance(n.child, parser.UNARY) and n.child.name == SELECTION: + while isinstance(n, parser.Unary) and n.name == SELECTION and isinstance(n.child, parser.Unary) and n.child.name == SELECTION: changes += 1 prop = n.prop @@ -147,7 +147,7 @@ def down_to_unions_subtractions_intersections(n: parser.Node) -> Tuple[parser.No ''' changes = 0 _o = (UNION, DIFFERENCE, INTERSECTION) - if isinstance(n, parser.UNARY) and n.name == SELECTION and n.child.name in _o: + if isinstance(n, parser.Unary) and n.name == SELECTION and n.child.name in _o: l = parser.Unary(SELECTION, n.prop, n.child.left) r = parser.Unary(SELECTION, n.prop, n.child.right) @@ -170,7 +170,7 @@ def duplicated_projection(n: parser.Node) -> Tuple[parser.Node, int]: def selection_inside_projection(n: parser.Node) -> Tuple[parser.Node, int]: '''This function locates things like σ j (π k(R)) and converts them into π k(σ j (R))''' - if isinstance(n, parser.UNARY) and n.name == SELECTION and n.child.name == PROJECTION: + if isinstance(n, parser.Unary) and n.name == SELECTION and n.child.name == PROJECTION: child = parser.Unary( SELECTION, n.prop, @@ -338,7 +338,7 @@ def swap_rename_select(n: parser.Node) -> int: Renaming the attributes used in the selection, so the operation is still valid.''' - if isinstance(n, parser.UNARY) and n.name == SELECTION and n.child.name == RENAME: + if isinstance(n, parser.Unary) and n.name == SELECTION and n.child.name == RENAME: # This is an inverse mapping for the rename renames = {v: k for k, v in n.child.get_rename_prop().items()} @@ -438,7 +438,7 @@ def selection_and_product(n: parser.Node, rels: Dict[str, Relation]) -> parser.N σ l (σ j (R) * σ i (Q)). Where j contains only attributes belonging to R, i contains attributes belonging to Q and l contains attributes belonging to both''' - if isinstance(n, parser.UNARY) and n.name == SELECTION and n.child.name in (PRODUCT, JOIN): + if isinstance(n, parser.Unary) and n.name == SELECTION and n.child.name in (PRODUCT, JOIN): l_attr = n.child.left.result_format(rels) r_attr = n.child.right.result_format(rels) From c5a71be509d20c28f6bb469bc67136d258ae9125 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Sun, 14 Jun 2020 22:43:50 +0200 Subject: [PATCH 07/26] Tell mypy about the values of name in nodes --- relational/parser.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/relational/parser.py b/relational/parser.py index 89862d0..41c0e98 100644 --- a/relational/parser.py +++ b/relational/parser.py @@ -24,7 +24,7 @@ # # Language definition here: # http://ltworf.github.io/relational/grammar.html -from typing import Optional, Union, List, Any, Dict +from typing import Optional, Union, List, Any, Dict, Literal from dataclasses import dataclass from relational import rtypes @@ -43,6 +43,11 @@ SELECTION = 'σ' RENAME = 'ρ' ARROW = '➡' +BINARY_LITERALS_T = Literal['*', '-', '∪', '∩', '÷', '⋈', '⧑', '⧒', '⧓'] + +UNARY_LITERALS_T = Literal['π', 'σ', 'ρ'] + + b_operators = (PRODUCT, DIFFERENCE, UNION, INTERSECTION, DIVISION, JOIN, JOIN_LEFT, JOIN_RIGHT, JOIN_FULL) # List of binary operators u_operators = (PROJECTION, SELECTION, RENAME) # List of unary operators @@ -194,6 +199,7 @@ class Variable(Node): @dataclass class Binary(Node): + name: BINARY_LITERALS_T left: Node right: Node @@ -214,6 +220,7 @@ class Binary(Node): @dataclass class Unary(Node): + name: UNARY_LITERALS_T prop: str child: Node From 9c3e012e71114fbba4033ef5ebfde5fbca32605d Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Sun, 14 Jun 2020 22:47:51 +0200 Subject: [PATCH 08/26] More type safety --- relational/optimizations.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/relational/optimizations.py b/relational/optimizations.py index 7a9ee62..7ee3268 100644 --- a/relational/optimizations.py +++ b/relational/optimizations.py @@ -148,6 +148,7 @@ def down_to_unions_subtractions_intersections(n: parser.Node) -> Tuple[parser.No changes = 0 _o = (UNION, DIFFERENCE, INTERSECTION) if isinstance(n, parser.Unary) and n.name == SELECTION and n.child.name in _o: + assert isinstance(n.child, parser.Binary) l = parser.Unary(SELECTION, n.prop, n.child.left) r = parser.Unary(SELECTION, n.prop, n.child.right) @@ -159,7 +160,7 @@ def duplicated_projection(n: parser.Node) -> Tuple[parser.Node, int]: '''This function locates thing like π i ( π j (R)) and replaces them with π i (R)''' - if n.name == PROJECTION and n.child.name == PROJECTION: + if isinstance(n, parser.Unary) and n.name == PROJECTION and isinstance(n.child, parser.Unary) and n.child.name == PROJECTION: return parser.Unary( PROJECTION, n.prop, @@ -170,7 +171,7 @@ def duplicated_projection(n: parser.Node) -> Tuple[parser.Node, int]: def selection_inside_projection(n: parser.Node) -> Tuple[parser.Node, int]: '''This function locates things like σ j (π k(R)) and converts them into π k(σ j (R))''' - if isinstance(n, parser.Unary) and n.name == SELECTION and n.child.name == PROJECTION: + if isinstance(n, parser.Unary) and n.name == SELECTION and isinstance(n.child, parser.Unary) and n.child.name == PROJECTION: child = parser.Unary( SELECTION, n.prop, @@ -305,7 +306,7 @@ def swap_rename_projection(n: parser.Node) -> Tuple[parser.Node, int]: Will also eliminate fields in the rename that are cut in the projection. ''' - if n.name == PROJECTION and n.child.name == RENAME: + if isinstance(n, parser.Unary) and n.name == PROJECTION and n.child.name == RENAME: # π index,name(ρ id➡index(R)) renames = n.child.get_rename_prop() projections = set(n.get_projection_prop()) @@ -515,7 +516,7 @@ def useless_projection(n: parser.Node, rels: Dict[str, Relation]) -> Tuple[parse ''' Removes projections that are over all the fields ''' - if n.name == PROJECTION and \ + if isinstance(n, parser.Unary) and n.name == PROJECTION and \ set(n.child.result_format(rels)) == set(i.strip() for i in n.prop.split(',')): return n.child, 1 From 16459868cf841a576371cb076a0f1a27113fe01b Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Sun, 14 Jun 2020 22:51:19 +0200 Subject: [PATCH 09/26] Import directly names from parser module --- relational/optimizations.py | 90 +++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 50 deletions(-) diff --git a/relational/optimizations.py b/relational/optimizations.py index 7ee3268..a091f6c 100644 --- a/relational/optimizations.py +++ b/relational/optimizations.py @@ -34,25 +34,15 @@ from typing import Tuple, Dict from relational.relation import Relation from relational import parser +from relational.parser import Binary, Unary, PRODUCT, \ + DIFFERENCE, UNION, INTERSECTION, DIVISION, JOIN, \ + JOIN_LEFT, JOIN_RIGHT, JOIN_FULL, PROJECTION, \ + SELECTION, RENAME, ARROW sel_op = ( '//=', '**=', 'and', 'not', 'in', '//', '**', '<<', '>>', '==', '!=', '>=', '<=', '+=', '-=', '*=', '/=', '%=', 'or', '+', '-', '*', '/', '&', '|', '^', '~', '<', '>', '%', '=', '(', ')', ',', '[', ']') -PRODUCT = parser.PRODUCT -DIFFERENCE = parser.DIFFERENCE -UNION = parser.UNION -INTERSECTION = parser.INTERSECTION -DIVISION = parser.DIVISION -JOIN = parser.JOIN -JOIN_LEFT = parser.JOIN_LEFT -JOIN_RIGHT = parser.JOIN_RIGHT -JOIN_FULL = parser.JOIN_FULL -PROJECTION = parser.PROJECTION -SELECTION = parser.SELECTION -RENAME = parser.RENAME -ARROW = parser.ARROW - def find_duplicates(node, dups=None): ''' @@ -72,7 +62,7 @@ def duplicated_select(n: parser.Node) -> Tuple[parser.Node, int]: in and ''' changes = 0 - while isinstance(n, parser.Unary) and n.name == SELECTION and isinstance(n.child, parser.Unary) and n.child.name == SELECTION: + while isinstance(n, Unary) and n.name == SELECTION and isinstance(n.child, Unary) and n.child.name == SELECTION: changes += 1 prop = n.prop @@ -82,7 +72,7 @@ def duplicated_select(n: parser.Node) -> Tuple[parser.Node, int]: # This adds parenthesis if they are needed if n.child.prop.startswith('(') or n.prop.startswith('('): prop = '(%s)' % prop - n = parser.Unary( + n = Unary( SELECTION, prop, n.child.child, @@ -125,14 +115,14 @@ def futile_union_intersection_subtraction(n: parser.Node) -> Tuple[parser.Node, elif n.name == DIFFERENCE and \ n.right.name == SELECTION and \ n.right.child == n.left: - return parser.Unary( + return Unary( SELECTION, '(not (%s))' % n.right.prop, n.right.child), 1 # Subtraction of the same thing or with selection on the left child elif n.name == DIFFERENCE and (n.left == n.right or (n.left.name == SELECTION and n.left.child == n.right)): - return parser.Unary( + return Unary( SELECTION, 'False', n.get_left_leaf() @@ -147,12 +137,12 @@ def down_to_unions_subtractions_intersections(n: parser.Node) -> Tuple[parser.No ''' changes = 0 _o = (UNION, DIFFERENCE, INTERSECTION) - if isinstance(n, parser.Unary) and n.name == SELECTION and n.child.name in _o: - assert isinstance(n.child, parser.Binary) - l = parser.Unary(SELECTION, n.prop, n.child.left) - r = parser.Unary(SELECTION, n.prop, n.child.right) + if isinstance(n, Unary) and n.name == SELECTION and n.child.name in _o: + assert isinstance(n.child, Binary) + l = Unary(SELECTION, n.prop, n.child.left) + r = Unary(SELECTION, n.prop, n.child.right) - return parser.Binary(n.child.name, l, r), 1 + return Binary(n.child.name, l, r), 1 return n, 0 @@ -160,8 +150,8 @@ def duplicated_projection(n: parser.Node) -> Tuple[parser.Node, int]: '''This function locates thing like π i ( π j (R)) and replaces them with π i (R)''' - if isinstance(n, parser.Unary) and n.name == PROJECTION and isinstance(n.child, parser.Unary) and n.child.name == PROJECTION: - return parser.Unary( + if isinstance(n, Unary) and n.name == PROJECTION and isinstance(n.child, Unary) and n.child.name == PROJECTION: + return Unary( PROJECTION, n.prop, n.child.child), 1 @@ -171,14 +161,14 @@ def duplicated_projection(n: parser.Node) -> Tuple[parser.Node, int]: def selection_inside_projection(n: parser.Node) -> Tuple[parser.Node, int]: '''This function locates things like σ j (π k(R)) and converts them into π k(σ j (R))''' - if isinstance(n, parser.Unary) and n.name == SELECTION and isinstance(n.child, parser.Unary) and n.child.name == PROJECTION: - child = parser.Unary( + if isinstance(n, Unary) and n.name == SELECTION and isinstance(n.child, Unary) and n.child.name == PROJECTION: + child = Unary( SELECTION, n.prop, n.child.child ) - return parser.Unary(PROJECTION, n.child.prop, child), 0 + return Unary(PROJECTION, n.child.prop, child), 0 return n, 0 @@ -192,8 +182,8 @@ def swap_union_renames(n: parser.Node) -> Tuple[parser.Node, int]: l_vars = n.left.get_rename_prop() r_vars = n.right.get_rename_prop() if r_vars == l_vars: - child = parser.Binary(n.name, n.left.child, n.right.child) - return parser.Unary(RENAME, n.left.prop, child), 1 + child = Binary(n.name, n.left.child, n.right.child) + return Unary(RENAME, n.left.prop, child), 1 return n, 0 @@ -231,7 +221,7 @@ def subsequent_renames(n: parser.Node) -> Tuple[parser.Node, int]: # Located two nested renames. prop = n.prop + ',' + n.child.prop child = n.child.child - n = parser.Unary(RENAME, prop, child) + n = Unary(RENAME, prop, child) # Creating a dictionary with the attributes renames = n.get_rename_prop() @@ -306,7 +296,7 @@ def swap_rename_projection(n: parser.Node) -> Tuple[parser.Node, int]: Will also eliminate fields in the rename that are cut in the projection. ''' - if isinstance(n, parser.Unary) and n.name == PROJECTION and n.child.name == RENAME: + if isinstance(n, Unary) and n.name == PROJECTION and n.child.name == RENAME: # π index,name(ρ id➡index(R)) renames = n.child.get_rename_prop() projections = set(n.get_projection_prop()) @@ -322,9 +312,9 @@ def swap_rename_projection(n: parser.Node) -> Tuple[parser.Node, int]: if i not in projections: del renames[i] - child = parser.Unary(PROJECTION,'' , n.child.child) + child = Unary(PROJECTION,'' , n.child.child) child.set_projection_prop(projections) - n = parser.Unary(RENAME, '', child) + n = Unary(RENAME, '', child) n.set_rename_prop(renames) return n, 1 @@ -339,7 +329,7 @@ def swap_rename_select(n: parser.Node) -> int: Renaming the attributes used in the selection, so the operation is still valid.''' - if isinstance(n, parser.Unary) and n.name == SELECTION and n.child.name == RENAME: + if isinstance(n, Unary) and n.name == SELECTION and n.child.name == RENAME: # This is an inverse mapping for the rename renames = {v: k for k, v in n.child.get_rename_prop().items()} @@ -354,8 +344,8 @@ def swap_rename_select(n: parser.Node) -> int: if len(splitted) > 1: tokens[i] += '.' + splitted[1] - child = parser.Unary(SELECTION, ' '.join(tokens), n.child.child) - return parser.Unary(RENAME, n.child.prop, child), 1 + child = Unary(SELECTION, ' '.join(tokens), n.child.child) + return Unary(RENAME, n.child.prop, child), 1 return n, 0 @@ -389,7 +379,7 @@ def select_union_intersect_subtract(n: parser.Node) -> int: prop = t_str % (n.left.prop, op, n.right.prop) else: prop = '%s %s %s' % (n.left.prop, op, n.right.prop) - return parser.Unary(SELECTION, prop, n.left.child), 1 + return Unary(SELECTION, prop, n.left.child), 1 return n, 0 @@ -403,13 +393,13 @@ def union_and_product(n: parser.Node) -> Tuple[parser.Node, int]: if n.left.left == n.right.left or n.left.left == n.right.right: l = n.left.right r = n.right.left if n.left.left == n.right.right else n.right.right - newchild = parser.Binary(UNION, l, r) - return parser.Binary(n.left.name, n.left.left, newchild), 1 + newchild = Binary(UNION, l, r) + return Binary(n.left.name, n.left.left, newchild), 1 elif n.left.right == n.right.left or n.left.left == n.right.right: l = n.left.left r = n.right.left if n.right.left == n.right.right else n.right.right - newchild = parser.Binary(UNION, l, r) - return parser.Binary(n.left.name, n.left.right, newchild), 1 + newchild = Binary(UNION, l, r) + return Binary(n.left.name, n.left.right, newchild), 1 return n, 0 @@ -429,8 +419,8 @@ def projection_and_union(n: parser.Node, rels: Dict[str, Relation]) -> Tuple[par n.right.name == PROJECTION and \ set(n.left.child.result_format(rels)) == set(n.right.child.result_format(rels)): - child = parser.Binary(UNION, n.left.child, n.right.child) - return parser.Unary(PROJECTION, n.right.prop, child), 0 + child = Binary(UNION, n.left.child, n.right.child) + return Unary(PROJECTION, n.right.prop, child), 0 return n, 0 @@ -439,7 +429,7 @@ def selection_and_product(n: parser.Node, rels: Dict[str, Relation]) -> parser.N σ l (σ j (R) * σ i (Q)). Where j contains only attributes belonging to R, i contains attributes belonging to Q and l contains attributes belonging to both''' - if isinstance(n, parser.Unary) and n.name == SELECTION and n.child.name in (PRODUCT, JOIN): + if isinstance(n, Unary) and n.name == SELECTION and n.child.name in (PRODUCT, JOIN): l_attr = n.child.left.result_format(rels) r_attr = n.child.right.result_format(rels) @@ -484,7 +474,7 @@ def selection_and_product(n: parser.Node, rels: Dict[str, Relation]) -> parser.N l_prop = ' and '.join((' '.join(i) for i in left)) if '(' in l_prop: l_prop = '(%s)' % l_prop - l_node = parser.Unary(SELECTION, l_prop, n.child.left) + l_node = Unary(SELECTION, l_prop, n.child.left) else: l_node = n.child.left @@ -493,18 +483,18 @@ def selection_and_product(n: parser.Node, rels: Dict[str, Relation]) -> parser.N r_prop = ' and '.join((' '.join(i) for i in right)) if '(' in r_prop: r_prop = '(%s)' % r_prop - r_node = parser.Unary(SELECTION, r_prop, n.child.right) + r_node = Unary(SELECTION, r_prop, n.child.right) else: r_node = n.child.right - b_node = parser.Binary(n.child.name, l_node, r_node) + b_node = Binary(n.child.name, l_node, r_node) # Changing main selection if both: both_prop = ' and '.join((' '.join(i) for i in both)) if '(' in both_prop: both_prop = '(%s)' % both_prop - r = parser.Unary(SELECTION, both_prop, b_node) + r = Unary(SELECTION, both_prop, b_node) return r, len(left) + len(right) else: # No need for general select return b_node, 1 @@ -516,7 +506,7 @@ def useless_projection(n: parser.Node, rels: Dict[str, Relation]) -> Tuple[parse ''' Removes projections that are over all the fields ''' - if isinstance(n, parser.Unary) and n.name == PROJECTION and \ + if isinstance(n, Unary) and n.name == PROJECTION and \ set(n.child.result_format(rels)) == set(i.strip() for i in n.prop.split(',')): return n.child, 1 From e35fc0f4bd43c3bc4adb4818099efc3d41802f3b Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Sun, 14 Jun 2020 22:54:58 +0200 Subject: [PATCH 10/26] More type safety --- relational/optimizations.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/relational/optimizations.py b/relational/optimizations.py index a091f6c..9eae8a3 100644 --- a/relational/optimizations.py +++ b/relational/optimizations.py @@ -195,7 +195,7 @@ def futile_renames(n: parser.Node) -> Tuple[parser.Node, int]: or removes the operation entirely if they all get removed ''' - if n.name == RENAME: + if isinstance(n, Unary) and n.name == RENAME: renames = n.get_rename_prop() changes = False for k, v in renames.items(): @@ -217,7 +217,10 @@ def subsequent_renames(n: parser.Node) -> Tuple[parser.Node, int]: into ρ ... (A) ''' - if n.name == RENAME and n.child.name == RENAME: + if isinstance(n, Unary) and \ + n.name == RENAME and \ + isinstance(n.child, Unary) and \ + n.child.name == RENAME: # Located two nested renames. prop = n.prop + ',' + n.child.prop child = n.child.child @@ -296,7 +299,10 @@ def swap_rename_projection(n: parser.Node) -> Tuple[parser.Node, int]: Will also eliminate fields in the rename that are cut in the projection. ''' - if isinstance(n, Unary) and n.name == PROJECTION and n.child.name == RENAME: + if isinstance(n, Unary) and \ + n.name == PROJECTION and \ + isinstance(n.child, Unary) and \ + n.child.name == RENAME: # π index,name(ρ id➡index(R)) renames = n.child.get_rename_prop() projections = set(n.get_projection_prop()) @@ -329,7 +335,10 @@ def swap_rename_select(n: parser.Node) -> int: Renaming the attributes used in the selection, so the operation is still valid.''' - if isinstance(n, Unary) and n.name == SELECTION and n.child.name == RENAME: + if isinstance(n, Unary) and \ + n.name == SELECTION and \ + isinstance(n.child, Unary) and \ + n.child.name == RENAME: # This is an inverse mapping for the rename renames = {v: k for k, v in n.child.get_rename_prop().items()} @@ -355,8 +364,11 @@ def select_union_intersect_subtract(n: parser.Node) -> int: and replaces them with σ (i OR q) (a) Removing a O(n²) operation like the union''' - if n.name in {UNION, INTERSECTION, DIFFERENCE} and \ + if isinstance(n, Binary) and \ + n.name in {UNION, INTERSECTION, DIFFERENCE} and \ + isinstance(n.left, Unary) and \ n.left.name == SELECTION and \ + isinstance(n.right, Unary) and \ n.right.name == SELECTION and \ n.left.child == n.right.child: @@ -388,7 +400,12 @@ def union_and_product(n: parser.Node) -> Tuple[parser.Node, int]: A * B ∪ A * C = A * (B ∪ C) Same thing with inner join ''' - if n.name == UNION and n.left.name in {PRODUCT, JOIN} and n.left.name == n.right.name: + if isinstance(n, Binary) and \ + n.name == UNION and \ + isinstance(n.left, Binary) and \ + n.left.name in {PRODUCT, JOIN} and \ + isinstance(n.right, Binary) and \ + n.left.name == n.right.name: if n.left.left == n.right.left or n.left.left == n.right.right: l = n.left.right @@ -429,7 +446,9 @@ def selection_and_product(n: parser.Node, rels: Dict[str, Relation]) -> parser.N σ l (σ j (R) * σ i (Q)). Where j contains only attributes belonging to R, i contains attributes belonging to Q and l contains attributes belonging to both''' - if isinstance(n, Unary) and n.name == SELECTION and n.child.name in (PRODUCT, JOIN): + if isinstance(n, Unary) and n.name == SELECTION and \ + isinstance(n.child, Binary) and \ + n.child.name in (PRODUCT, JOIN): l_attr = n.child.left.result_format(rels) r_attr = n.child.right.result_format(rels) From 95a4287e68c26d29008da47b8abde3b7869a286b Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Wed, 17 Jun 2020 23:05:27 +0200 Subject: [PATCH 11/26] More of the same --- relational/optimizations.py | 45 +++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/relational/optimizations.py b/relational/optimizations.py index 9eae8a3..00f91c0 100644 --- a/relational/optimizations.py +++ b/relational/optimizations.py @@ -91,29 +91,34 @@ def futile_union_intersection_subtraction(n: parser.Node) -> Tuple[parser.Node, σ k (R) ∩ R --> σ k (R) ''' - changes = 0 + if not isinstance(n, Binary): + return n, 0 # Union and intersection of the same thing if n.name in (UNION, INTERSECTION, JOIN, JOIN_LEFT, JOIN_RIGHT, JOIN_FULL) and n.left == n.right: return n.left, 1 # selection and union of the same thing - elif (n.name == UNION): - if n.left.name == SELECTION and n.left.child == n.right: + elif n.name == UNION: + if n.left.name == SELECTION and isinstance(n.left, Unary) and n.left.child == n.right: return n.right, 1 - elif n.right.name == SELECTION and n.right.child == n.left: + elif n.right.name == SELECTION and isinstance(n.right, Unary) and n.right.child == n.left: return n.left, 1 # selection and intersection of the same thing elif n.name == INTERSECTION: if n.left.name == SELECTION and n.left.child == n.right: return n.left, 1 - elif n.right.name == SELECTION and n.right.child == n.left: + elif n.right.name == SELECTION and \ + isinstance(n.right, Unary) and \ + n.right.child == n.left: return n.right, 1 # Subtraction and selection of the same thing elif n.name == DIFFERENCE and \ + isinstance(n, Binary) and \ n.right.name == SELECTION and \ + isinstance(n.right, Unary) and \ n.right.child == n.left: return Unary( SELECTION, @@ -178,7 +183,12 @@ def swap_union_renames(n: parser.Node) -> Tuple[parser.Node, int]: and replaces them with ρ a➡b(R ∪ Q). Does the same with subtraction and intersection''' - if n.name in (DIFFERENCE, UNION, INTERSECTION) and n.left.name == RENAME and n.right.name == RENAME: + if n.name in (DIFFERENCE, UNION, INTERSECTION) and \ + isinstance(n, Binary) and \ + n.left.name == RENAME and \ + isinstance(n.left, Unary) and\ + n.right.name == RENAME and \ + isinstance(n.right, Unary): l_vars = n.left.get_rename_prop() r_vars = n.right.get_rename_prop() if r_vars == l_vars: @@ -327,7 +337,7 @@ def swap_rename_projection(n: parser.Node) -> Tuple[parser.Node, int]: return n, 0 -def swap_rename_select(n: parser.Node) -> int: +def swap_rename_select(n: parser.Node) -> Tuple[parser.Node, int]: '''This function locates things like σ k(ρ j(R)) and replaces them with @@ -358,20 +368,19 @@ def swap_rename_select(n: parser.Node) -> int: return n, 0 -def select_union_intersect_subtract(n: parser.Node) -> int: +def select_union_intersect_subtract(n: parser.Node) -> Tuple[parser.Node, int]: '''This function locates things like σ i(a) ∪ σ q(a) and replaces them with σ (i OR q) (a) Removing a O(n²) operation like the union''' if isinstance(n, Binary) and \ - n.name in {UNION, INTERSECTION, DIFFERENCE} and \ - isinstance(n.left, Unary) and \ - n.left.name == SELECTION and \ - isinstance(n.right, Unary) and \ - n.right.name == SELECTION and \ - n.left.child == n.right.child: - + n.name in {UNION, INTERSECTION, DIFFERENCE} and \ + isinstance(n.left, Unary) and \ + n.left.name == SELECTION and \ + isinstance(n.right, Unary) and \ + n.right.name == SELECTION and \ + n.left.child == n.right.child: d = {UNION: 'or', INTERSECTION: 'and', DIFFERENCE: 'and not'} op = d[n.name] @@ -432,16 +441,18 @@ def projection_and_union(n: parser.Node, rels: Dict[str, Relation]) -> Tuple[par ''' changes = 0 if n.name == UNION and \ + isinstance(n, Binary) and \ n.left.name == PROJECTION and \ + isinstance(n.left, Unary) and \ n.right.name == PROJECTION and \ + isinstance(n.right, Unary) and \ set(n.left.child.result_format(rels)) == set(n.right.child.result_format(rels)): - child = Binary(UNION, n.left.child, n.right.child) return Unary(PROJECTION, n.right.prop, child), 0 return n, 0 -def selection_and_product(n: parser.Node, rels: Dict[str, Relation]) -> parser.Node: +def selection_and_product(n: parser.Node, rels: Dict[str, Relation]) -> Tuple[parser.Node, int]: '''This function locates things like σ k (R*Q) and converts them into σ l (σ j (R) * σ i (Q)). Where j contains only attributes belonging to R, i contains attributes belonging to Q and l contains attributes belonging to both''' From 79610ab8f5570096027d9f67d428f082fb172900 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Thu, 18 Jun 2020 16:59:15 +0200 Subject: [PATCH 12/26] More of the same --- relational/optimizations.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/relational/optimizations.py b/relational/optimizations.py index 00f91c0..67e69ec 100644 --- a/relational/optimizations.py +++ b/relational/optimizations.py @@ -126,7 +126,9 @@ def futile_union_intersection_subtraction(n: parser.Node) -> Tuple[parser.Node, n.right.child), 1 # Subtraction of the same thing or with selection on the left child - elif n.name == DIFFERENCE and (n.left == n.right or (n.left.name == SELECTION and n.left.child == n.right)): + elif n.name == DIFFERENCE and \ + isinstance(n, Binary) and \ + (n.left == n.right or (n.left.name == SELECTION and n.left.child == n.right)): return Unary( SELECTION, 'False', @@ -329,7 +331,7 @@ def swap_rename_projection(n: parser.Node) -> Tuple[parser.Node, int]: del renames[i] child = Unary(PROJECTION,'' , n.child.child) - child.set_projection_prop(projections) + child.set_projection_prop(list(projections)) n = Unary(RENAME, '', child) n.set_rename_prop(renames) return n, 1 From 9f171fa393169d895df14d56c70e1074480dee12 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Sat, 27 Jun 2020 15:47:55 +0200 Subject: [PATCH 13/26] Check instance once --- relational/parser.py | 47 +++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/relational/parser.py b/relational/parser.py index 41c0e98..efe4015 100644 --- a/relational/parser.py +++ b/relational/parser.py @@ -147,29 +147,32 @@ class Node: if isinstance(self, Variable): #FIXME this is ugly return list(rels[self.name].header) - elif isinstance(self, Binary) and self.name in (DIFFERENCE, UNION, INTERSECTION): - return self.left.result_format(rels) - elif isinstance(self, Binary) and self.name == DIVISION: - return list(set(self.left.result_format(rels)) - set(self.right.result_format(rels))) - elif self.name == PROJECTION: - return self.get_projection_prop() - elif self.name == PRODUCT: - return self.left.result_format(rels) + self.right.result_format(rels) - elif self.name == SELECTION: - return self.child.result_format(rels) - elif self.name == RENAME: - _vars = {} - for i in self.prop.split(','): - q = i.split(ARROW) - _vars[q[0].strip()] = q[1].strip() + elif isinstance(self, Binary): + if self.name in (DIFFERENCE, UNION, INTERSECTION): + return self.left.result_format(rels) + elif self.name == DIVISION: + return list(set(self.left.result_format(rels)) - set(self.right.result_format(rels))) + elif self.name == PRODUCT: + return self.left.result_format(rels) + self.right.result_format(rels) + elif self.name in (JOIN, JOIN_LEFT, JOIN_RIGHT, JOIN_FULL): + return list(set(self.left.result_format(rels)).union(set(self.right.result_format(rels)))) + elif isinstance(self, Unary): + if self.name == PROJECTION: + return self.get_projection_prop() + elif self.name == SELECTION: + return self.child.result_format(rels) + elif self.name == RENAME: + _vars = {} + for i in self.prop.split(','): + q = i.split(ARROW) + _vars[q[0].strip()] = q[1].strip() + + _fields = self.child.result_format(rels) + for i in range(len(_fields)): + if _fields[i] in _vars: + _fields[i] = _vars[_fields[i]] + return _fields - _fields = self.child.result_format(rels) - for i in range(len(_fields)): - if _fields[i] in _vars: - _fields[i] = _vars[_fields[i]] - return _fields - elif self.name in (JOIN, JOIN_LEFT, JOIN_RIGHT, JOIN_FULL): - return list(set(self.left.result_format(rels)).union(set(self.right.result_format(rels)))) raise ValueError('What kind of alien object is this?') def __eq__(self, other): #FIXME From e5dad023970fbd2264a03bf33820f6bdd78cb274 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Sat, 27 Jun 2020 15:49:19 +0200 Subject: [PATCH 14/26] Remove duplicate code --- relational/parser.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/relational/parser.py b/relational/parser.py index efe4015..017d905 100644 --- a/relational/parser.py +++ b/relational/parser.py @@ -162,11 +162,7 @@ class Node: elif self.name == SELECTION: return self.child.result_format(rels) elif self.name == RENAME: - _vars = {} - for i in self.prop.split(','): - q = i.split(ARROW) - _vars[q[0].strip()] = q[1].strip() - + _vars = self.get_rename_prop() _fields = self.child.result_format(rels) for i in range(len(_fields)): if _fields[i] in _vars: From f21a7beb414b3e9e60993aa47604c93447ccf25e Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Sat, 27 Jun 2020 15:51:00 +0200 Subject: [PATCH 15/26] Assert to make mypy happy --- relational/parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/relational/parser.py b/relational/parser.py index 017d905..c642f9c 100644 --- a/relational/parser.py +++ b/relational/parser.py @@ -287,6 +287,7 @@ def parse_tokens(expression: List[Union[list, str]]) -> Node: # The list contains only 1 string. Means it is the name of a relation if len(expression) == 1: + assert isinstance(expression[0], str) if not rtypes.is_valid_relation_name(expression[0]): raise ParserException( f'{expression[0]!r} is not a valid relation name') From 40a15178fe5726e32ef06a881ec4516ca3c0f018 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Wed, 12 Aug 2020 10:52:11 +0200 Subject: [PATCH 16/26] Make mypy happy --- relational/optimizer.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/relational/optimizer.py b/relational/optimizer.py index 8976f31..05a23eb 100644 --- a/relational/optimizer.py +++ b/relational/optimizer.py @@ -72,24 +72,19 @@ def optimize_all(expression: Union[str, Node], rels: Dict[str, Relation], specif else: raise (TypeError("expression must be a string or a node")) - if isinstance(debug, list): - dbg = True - else: - dbg = False - total = 1 while total != 0: total = 0 if specific: for i in optimizations.specific_optimizations: n, c = recursive_scan(i, n, rels) - if c != 0 and dbg: + if c != 0 and isinstance(debug, list): debug.append(str(n)) total += c if general: for i in optimizations.general_optimizations: n, c = recursive_scan(i, n, None) - if c != 0 and dbg: + if c != 0 and isinstance(debug, list): debug.append(str(n)) total += c if tostr: From 786a9d61f53fe6fc9e703a44b80a23d07b02e6d6 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Wed, 12 Aug 2020 11:05:47 +0200 Subject: [PATCH 17/26] Rework the tokenizing function --- relational/optimizations.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/relational/optimizations.py b/relational/optimizations.py index 67e69ec..afeca55 100644 --- a/relational/optimizations.py +++ b/relational/optimizations.py @@ -30,7 +30,7 @@ from io import StringIO from tokenize import generate_tokens -from typing import Tuple, Dict +from typing import Tuple, Dict, List from relational.relation import Relation from relational import parser @@ -263,11 +263,11 @@ def subsequent_renames(n: parser.Node) -> Tuple[parser.Node, int]: return n, 0 -class level_string(str): +class LevelString(str): level = 0 -def tokenize_select(expression): +def tokenize_select(expression: str) -> List[LevelString]: '''This function returns the list of tokens present in a selection. The expression can contain parenthesis. It will use a subclass of str with the attribute level, which @@ -275,8 +275,6 @@ def tokenize_select(expression): g = generate_tokens(StringIO(str(expression)).readline) l = list(token[1] for token in g) - l.remove('') - # Changes the 'a','.','method' token group into a single 'a.method' token try: while True: @@ -287,17 +285,21 @@ def tokenize_select(expression): except: pass + r = [] level = 0 - for i in range(len(l)): - l[i] = level_string(l[i]) - l[i].level = level + for i in l: + if not i: + continue + value = LevelString(i) + value.level = level - if l[i] == '(': + if value == '(': level += 1 - elif l[i] == ')': + elif value == ')': level -= 1 + r.append(value) - return l + return r def swap_rename_projection(n: parser.Node) -> Tuple[parser.Node, int]: From cabae01f7cdca8bde8f1cccda384b0b289973ec4 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Wed, 12 Aug 2020 15:10:42 +0200 Subject: [PATCH 18/26] Happify mypy --- relational/optimizations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/relational/optimizations.py b/relational/optimizations.py index afeca55..7c00ee8 100644 --- a/relational/optimizations.py +++ b/relational/optimizations.py @@ -363,9 +363,9 @@ def swap_rename_select(n: parser.Node) -> Tuple[parser.Node, int]: for i in range(len(tokens)): splitted = tokens[i].split('.', 1) if splitted[0] in renames: - tokens[i] = renames[splitted[0]] + tokens[i] = LevelString(renames[splitted[0]]) if len(splitted) > 1: - tokens[i] += '.' + splitted[1] + tokens[i] = LevelString(tokens[i] + '.' + splitted[1]) child = Unary(SELECTION, ' '.join(tokens), n.child.child) return Unary(RENAME, n.child.prop, child), 1 From 8d49b393c869995a75a08de50a92b0882a09e9c0 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Wed, 12 Aug 2020 15:16:58 +0200 Subject: [PATCH 19/26] Give up on the literals mypy is too stupid to infer them --- relational/parser.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/relational/parser.py b/relational/parser.py index c642f9c..e95d27f 100644 --- a/relational/parser.py +++ b/relational/parser.py @@ -43,10 +43,6 @@ SELECTION = 'σ' RENAME = 'ρ' ARROW = '➡' -BINARY_LITERALS_T = Literal['*', '-', '∪', '∩', '÷', '⋈', '⧑', '⧒', '⧓'] - -UNARY_LITERALS_T = Literal['π', 'σ', 'ρ'] - b_operators = (PRODUCT, DIFFERENCE, UNION, INTERSECTION, DIVISION, JOIN, JOIN_LEFT, JOIN_RIGHT, JOIN_FULL) # List of binary operators @@ -198,7 +194,7 @@ class Variable(Node): @dataclass class Binary(Node): - name: BINARY_LITERALS_T + name: str left: Node right: Node @@ -219,7 +215,7 @@ class Binary(Node): @dataclass class Unary(Node): - name: UNARY_LITERALS_T + name: str prop: str child: Node From bd58442912cf13e7f3795821e2c13edf5e23aed1 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Wed, 12 Aug 2020 15:20:29 +0200 Subject: [PATCH 20/26] Make hash raise an error --- relational/relation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/relational/relation.py b/relational/relation.py index 346730d..84782dd 100644 --- a/relational/relation.py +++ b/relational/relation.py @@ -52,7 +52,8 @@ class Relation: An empty relation needs a header, and can be filled using the insert() method. ''' - __hash__ = None # type: None + def __hash__(self): + raise NotImplementedError() def __init__(self, filename: Optional[Union[str, Path]] = None) -> None: self._readonly = False From be932abcaaaca27da876a78f2db731819bde065d Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Wed, 12 Aug 2020 15:26:41 +0200 Subject: [PATCH 21/26] More isinstance --- relational/optimizations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/relational/optimizations.py b/relational/optimizations.py index 7c00ee8..d7d20db 100644 --- a/relational/optimizations.py +++ b/relational/optimizations.py @@ -107,7 +107,7 @@ def futile_union_intersection_subtraction(n: parser.Node) -> Tuple[parser.Node, # selection and intersection of the same thing elif n.name == INTERSECTION: - if n.left.name == SELECTION and n.left.child == n.right: + if n.left.name == SELECTION and isinstance(n.left, Unary) and n.left.child == n.right: return n.left, 1 elif n.right.name == SELECTION and \ isinstance(n.right, Unary) and \ @@ -128,7 +128,7 @@ def futile_union_intersection_subtraction(n: parser.Node) -> Tuple[parser.Node, # Subtraction of the same thing or with selection on the left child elif n.name == DIFFERENCE and \ isinstance(n, Binary) and \ - (n.left == n.right or (n.left.name == SELECTION and n.left.child == n.right)): + (n.left == n.right or (n.left.name == SELECTION and isinstance(n.left, Unary) and n.left.child == n.right)): return Unary( SELECTION, 'False', From f7f51b34137b56380ec7de304b7e5deb224f83a8 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Wed, 12 Aug 2020 15:26:56 +0200 Subject: [PATCH 22/26] Make mypy happy --- relational/parser.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/relational/parser.py b/relational/parser.py index e95d27f..143edf5 100644 --- a/relational/parser.py +++ b/relational/parser.py @@ -309,7 +309,7 @@ def parse_tokens(expression: List[Union[list, str]]) -> Node: if len(expression[i + 1:]) == 0: raise ParserException( f'Expected right operand for {expression[i]!r}') - return Binary(expression[i], parse_tokens(expression[:i]), parse_tokens(expression[i + 1:])) + return Binary(expression[i], parse_tokens(expression[:i]), parse_tokens(expression[i + 1:])) # type: ignore '''Searches for unary operators, parsing from right to left''' for i in range(len(expression) - 1, -1, -1): if expression[i] in u_operators: # Unary operator @@ -318,9 +318,9 @@ def parse_tokens(expression: List[Union[list, str]]) -> Node: f'Expected more tokens in {expression[i]!r}') return Unary( - expression[i], - prop=expression[1 + i].strip(), - child=parse_tokens(expression[2 + i]) + expression[i], # type: ignore + prop=expression[1 + i].strip(), # type: ignore + child=parse_tokens(expression[2 + i]) # type: ignore ) raise ParserException(f'Parse error on {expression!r}') From 6fcb5ecb85a4921c4aa6a367590ba06215dc2f6d Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Wed, 12 Aug 2020 15:33:41 +0200 Subject: [PATCH 23/26] Bring more happyness to mypy --- relational/optimizations.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/relational/optimizations.py b/relational/optimizations.py index d7d20db..a22aadf 100644 --- a/relational/optimizations.py +++ b/relational/optimizations.py @@ -34,7 +34,7 @@ from typing import Tuple, Dict, List from relational.relation import Relation from relational import parser -from relational.parser import Binary, Unary, PRODUCT, \ +from relational.parser import Binary, Unary, Node, PRODUCT, \ DIFFERENCE, UNION, INTERSECTION, DIVISION, JOIN, \ JOIN_LEFT, JOIN_RIGHT, JOIN_FULL, PROJECTION, \ SELECTION, RENAME, ARROW @@ -468,8 +468,8 @@ def selection_and_product(n: parser.Node, rels: Dict[str, Relation]) -> Tuple[pa r_attr = n.child.right.result_format(rels) tokens = tokenize_select(n.prop) - groups = [] - temp = [] + groups: List[List[LevelString]] = [] + temp: List[LevelString] = [] for i in tokens: if i == 'and' and i.level == 0: @@ -477,9 +477,9 @@ def selection_and_product(n: parser.Node, rels: Dict[str, Relation]) -> Tuple[pa temp = [] else: temp.append(i) - if len(temp) != 0: + if len(temp): groups.append(temp) - temp = [] + del temp left = [] right = [] @@ -508,7 +508,7 @@ def selection_and_product(n: parser.Node, rels: Dict[str, Relation]) -> Tuple[pa l_prop = ' and '.join((' '.join(i) for i in left)) if '(' in l_prop: l_prop = '(%s)' % l_prop - l_node = Unary(SELECTION, l_prop, n.child.left) + l_node: Node = Unary(SELECTION, l_prop, n.child.left) else: l_node = n.child.left @@ -517,7 +517,7 @@ def selection_and_product(n: parser.Node, rels: Dict[str, Relation]) -> Tuple[pa r_prop = ' and '.join((' '.join(i) for i in right)) if '(' in r_prop: r_prop = '(%s)' % r_prop - r_node = Unary(SELECTION, r_prop, n.child.right) + r_node: Node = Unary(SELECTION, r_prop, n.child.right) else: r_node = n.child.right From fbef0f9deb07516822f69ea7526841d4d0292821 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Wed, 12 Aug 2020 17:32:41 +0200 Subject: [PATCH 24/26] Make mypy happy --- relational/optimizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/relational/optimizer.py b/relational/optimizer.py index 05a23eb..b3b04ee 100644 --- a/relational/optimizer.py +++ b/relational/optimizer.py @@ -82,8 +82,8 @@ def optimize_all(expression: Union[str, Node], rels: Dict[str, Relation], specif debug.append(str(n)) total += c if general: - for i in optimizations.general_optimizations: - n, c = recursive_scan(i, n, None) + for j in optimizations.general_optimizations: + n, c = recursive_scan(j, n, None) if c != 0 and isinstance(debug, list): debug.append(str(n)) total += c From 731327c079afc3e1417a3dafd76eaa52dd34fb02 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Wed, 12 Aug 2020 17:34:46 +0200 Subject: [PATCH 25/26] Var rename --- relational/optimizations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/relational/optimizations.py b/relational/optimizations.py index a22aadf..ad4e735 100644 --- a/relational/optimizations.py +++ b/relational/optimizations.py @@ -471,12 +471,12 @@ def selection_and_product(n: parser.Node, rels: Dict[str, Relation]) -> Tuple[pa groups: List[List[LevelString]] = [] temp: List[LevelString] = [] - for i in tokens: - if i == 'and' and i.level == 0: + for k in tokens: + if k == 'and' and k.level == 0: groups.append(temp) temp = [] else: - temp.append(i) + temp.append(k) if len(temp): groups.append(temp) del temp From 2ceaa3e1db28e766a27e64c2ab96eabab91bb203 Mon Sep 17 00:00:00 2001 From: Salvo 'LtWorf' Tomaselli Date: Wed, 12 Aug 2020 17:35:55 +0200 Subject: [PATCH 26/26] The final rename --- relational/optimizations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/relational/optimizations.py b/relational/optimizations.py index ad4e735..32a078e 100644 --- a/relational/optimizations.py +++ b/relational/optimizations.py @@ -490,10 +490,10 @@ def selection_and_product(n: parser.Node, rels: Dict[str, Relation]) -> Tuple[pa r_fields = False # has fields in left? for j in set(i).difference(sel_op): - j = j.split('.')[0] - if j in l_attr: # Field in left + t = j.split('.')[0] + if t in l_attr: # Field in left l_fields = True - if j in r_attr: # Field in right + if t in r_attr: # Field in right r_fields = True if l_fields and not r_fields: