tokenizer workingpython relational/optimizer.py | lesspython relational/optimizer.py | less

git-svn-id: http://galileo.dmi.unict.it/svn/relational/trunk@109 014f5005-505e-4b48-8d0a-63407b615a7c

relational/optimizer.py

@@ -153,46 +153,57 @@ def tokenize(expression):
     3 previous stuff was a unary operator
     4 previous stuff was a binary operator
     '''
-    
+
-    if expression.startswith('('): #Parenthesis state
+    while len(expression)>0:
-        state=2
+        print "Expression", expression
-        par_count=0 #Count of parenthesis
+        print "Items" ,items
-        end=0
+        if expression.startswith('('): #Parenthesis state
+            state=2
+            par_count=0 #Count of parenthesis
+            end=0
+            
+            for i in range(len(expression)):
+                if expression[i]=='(':
+                    par_count+=1
+                elif expression[i]==')':
+                    par_count-=1
+                    if par_count==0:
+                        end=i
+                        break
+            items.append(tokenize(expression[1:end]))
+            expression=expression[end+1:].strip()
         
-        for i in len(expression):
+        elif expression.startswith("σ") or expression.startswith("π") or expression.startswith("ρ"): #Unary
-            if expression[i]=='(':
+            items.append(expression[0:2]) #Adding operator in the top of the list
-                par_count+=1
+            expression=expression[2:].strip() #Removing operator from the expression
-            elif expression[i]==')':
+            par=expression.find('(')
-                par_count-=1
-                if par_count==0:
-                    end=i
-                    break
-        items.append(tokenize(expression[1:end]))
-        epression=expression[end+1:].strip()
         
-    elif expression.startswith("σ"): or expression.startswith("π") or expression.startswith("ρ"): #Unary
+            items.append(expression[:par]) #Inserting parameter of the operator
-        items.append(expression[0:2]) #Adding operator in the top of the list
+            expression=expression[par:].strip() #Removing parameter from the expression
-        
+        elif expression.startswith("*") or expression.startswith("-"):
-        expression=expression[2:].strip() #Removing operator from the expression
-        par=expression.find('(')
-        
-        items.append(expression[:par]) #Inserting parameter of the operator
-        expression=expression[par:].strip() #Removing parameter from the expression
-    elif expression.startswith("*") or expression.startswith("-"):
-        state=4
-    elif expression.startswith("ᑎ") or expression.startswith("ᑌ"): #Binary short
-        state=4
-    elif expression.startswith("ᐅ"): #Binary long
-        state=4
-    else: #Relation (hopefully)
-        if state==1: #Previous was a relation, appending to the last token
-            i=items.pop()
-            items.append(i+expression[0])
-            expression=expression[1:].strip() #1 char from the expression
-        else:
-            state=1
             items.append(expression[0])
             expression=expression[1:].strip() #1 char from the expression
+            state=4
+        elif expression.startswith("ᑎ") or expression.startswith("ᑌ"): #Binary short
+            items.append(expression[0:3]) #Adding operator in the top of the list
+            expression=expression[3:].strip() #Removing operator from the expression
+
+            state=4
+        elif expression.startswith("ᐅ"): #Binary long
+            i=expression.find("ᐊ")
+            items.append(expression[:i+3])
+            expression=expression[i+3:].strip()
+            
+            state=4
+        else: #Relation (hopefully)
+            if state==1: #Previous was a relation, appending to the last token
+                i=items.pop()
+                items.append(i+expression[0])
+                expression=expression[1:].strip() #1 char from the expression
+            else:
+                state=1
+                items.append(expression[0])
+                expression=expression[1:].strip() #1 char from the expression
     
     return items
 
@@ -200,5 +211,8 @@ def tokenize(expression):
 if __name__=="__main__":
     #n=node(u"((a ᑌ b) - c ᑌ d) - b")
     #n=node(u"((((((((((((2)))))))))))) - (3 * 5) - 2")
-    n=node(u"π a,b (d-a*b)")
+    #n=node(u"π a,b (d-a*b)")
-    print n.__str__()
+    
+    #print n.__str__()
+    print tokenize("((a ᑌ b) - c ᑌ d) ᐅRIGHTᐊ a * (π a,b (a))")
+    #print tokenize("(a)")