var testvar1 = 34.5;

var tokenCategories =
{
    operator:
    {
        name:   'operator',
        terminal: false,
        follow: ['number','string','function','open','variable'],
        start:  false,
        end:    false

    },
    comperator:
    {
        name:   'comperator',
        terminal: false,
        follow: ['number','string','open','function','variable'],
        start:  false,
        end:    false

    },
    logical:
    {
        name:   'logical',
        terminal: false,
        follow: ['number','string','function','open','variable'],
        start:  false,
        end:    false

    },
    number:
    {
        name:   'number',
        terminal: true,
        follow: ['logical','comperator','operator','close'],
        start: true,
        end:   true

    },
    func:
    {
        name:   'function',
        terminal: true,
        follow: ['logical','comperator','operator','close'],
        start: true,
        end:   true
    },
    variable:
    {
        name:   'variable',
        terminal: true,
        follow: ['open','logical','comperator','operator','close'],
        start: true,
        end:   true
    },
    string:
    {
        name:   'string',
        terminal: true,
        follow: ['logical','comperator','operator','close'],
        start: true,
        end:   true
    },
    open:
    {
        name:   'open',
        desc:   '(',
        terminal: true,
        follow: ['open','number','string','function','variable'],
        start:  true,
        end:    false
    },
    seperator:
    {
        name:   'seperator',
        desc:   ',',
        terminal:   false,
        follow: ['number','string','function','variable'],
        start: false,
        end:    false
    },
    close:
    {
        desc:   ')',
        name:   'close',
        terminal: true,
        follow: ['comperator','logical','operator','close'],
        start: false,
        end:   true
    }




}

var tokenTypes =
{
    add:
    {
        name: '+',
        category: tokenCategories.operator,
        weight:     3
    },
    subtract:
    {
        name: '-',
        category: tokenCategories.operator,
        weight:     3
    },
    multiply:
    {
        name: '*',
        category: tokenCategories.operator,
        weight:     4
    },
    divide:
    {
        name: '/',
        category: tokenCategories.operator,
        weight:     4
    },
    eq:
    {
        name: '=',
        category: tokenCategories.comperator,
        weight:     2
    },
    ne:
    {
        name: '!=',
        category: tokenCategories.comperator,
        weight:     2
    },
    gt:
    {
        name: '>',
        category: tokenCategories.comperator,
        weight:     2
    },
    lt:
    {
        name: '<',
        category: tokenCategories.comperator,
        weight:     2
    },
    gte:
    {
        name: '>=',
        category: tokenCategories.comperator,
        weight:     2
    },
    lte:
    {
        name: '<=',
        category: tokenCategories.comperator,
        weight:     2
    },
    logand:
    {
        name: '&&',
        category: tokenCategories.logical,
        weight:     1
    },
    logor:
    {
        name: '||',
        category: tokenCategories.logical,
        weight:     1
    }
}

function arrayIndexOf(a,v)
{
    r = -1;
    for (var i = 0; i < a.length; i++)
    {
        if (a[i]==v)
            return i;
    }

    return r;
}

function isArray(obj) {
   if (obj == undefined)
       return false;
   if (obj == null)
       return false;
   if (obj.constructor.toString().indexOf("Array") == -1)
      return false;
   else
      return true;
}
function TokenSyntaxTest(tokens)
{
    this.tokens = tokens;
    this.pos = 0;
    this.offset = 0;
    this.error = '';
    this.endedWith = ')';

    this.test = function(subSet,offset)
    {
        consoleLog('Entered testing');
        if (subSet == undefined)
        {
            subSet = false;
        }
        else
        {
            
            this.offset = offset;
        }

        
        var token = this.tokens[0];
        if (!this.testStart(token))
        {
            consoleLog('Syntax error at char ' + token.position + ': ' + token.getDescription());
            this.error = 'Syntax error at char ' + token.position + ': ' + token.getDescription();
            return -1;
        }
        while (this.pos+this.offset < this.tokens.length-1)
        {
            consoleLog('Start loop: ' + this.tokens[this.pos+this.offset].getDescription());
            if (!this.testFollow(this.tokens[this.pos+this.offset],this.peek(),subSet))
            {
                consoleLog('Syntax error at char ' + this.peek().position + ': ' + this.peek().getDescription());
                this.error = 'Syntax error at char ' + this.peek().position + ': ' + this.peek().getDescription();
                return -1;
            }
            if (this.tokens[this.pos+this.offset].type==tokenCategories.open)
            {
                if(this.offset+this.pos>0 && this.tokens[this.pos+this.offset-1].type.name=='variable')
                {
                    var priorPos = this.pos+this.offset-1;
                    this.tokens[priorPos].type = tokenCategories.func;
                }

                //var rest = this.tokens.slice(this.pos+1);
                var test = new TokenSyntaxTest(this.tokens);
                var endPos = test.test(true,this.pos+this.offset+1);
                consoleLog('Finished subset with endpos ' + endPos + ' and ended with ' + test.endedWith);
                if (endPos==-1)
                {
                    this.error = test.error;
                    return -1;
                }
                this.tokens.splice(this.pos+this.offset, endPos+1-this.pos-this.offset, this.tokens.slice(this.pos+this.offset+1,endPos));
                //alert(this.pos);
                while (test.endedWith==',')
                {
                    this.pos++;
                    test = new TokenSyntaxTest(this.tokens);
                    var endPos = test.test(true,this.pos+this.offset+1);
                    consoleLog('Finished subset with endpos ' + endPos + ' and ended with ' + test.endedWith);
                    if (endPos==-1)
                    {
                        this.error = test.error;
                        return -1;
                    }

                    this.tokens.splice(this.pos+this.offset, endPos+1-this.pos-this.offset, this.tokens.slice(this.pos+this.offset,endPos));
                }
                
            }
            else if (this.tokens[this.pos+this.offset].type==tokenCategories.close && subSet)
            {
                consoleLog('End bracket found at position ' + (this.pos+this.offset));
                return this.pos+this.offset;
            }
            else if (this.tokens[this.pos+this.offset].type==tokenCategories.seperator && subSet)
            {
                consoleLog('Seperator found at position ' + (this.pos+this.offset));
                this.endedWith = ',';
                return this.pos+this.offset;
            }
            else if(this.tokens[this.pos+this.offset].type==tokenCategories.close && !subSet)
            {
                consoleLog('Closing bracket without opening at char ' + token.position);
                this.error = 'Closing bracket without opening at char ' + token.position;
                return -1;
            }
            else if(this.tokens[this.pos+this.offset].type==tokenCategories.seperator && !subSet)
            {
                consoleLog('Closing bracket without opening at char ' + token.position);
                this.error = 'Closing bracket without opening at char ' + token.position;
                return -1;
            }
            this.pos++; 
        }
        if (this.pos+this.offset >= this.tokens.length)
        {
            return;
        }
        if (!this.testEnd(this.tokens[this.pos+this.offset]))
        {
            consoleLog('Syntax error at end char ' + token.position + ': ' + token.getDescription());
            this.error = 'Syntax error at end char ' + token.position + ': ' + token.getDescription();
            return -1;
        }
        if (this.tokens[this.pos+this.offset].type==tokenCategories.close && subSet)
        {
            consoleLog('End of subset with bracket at position ' + (this.pos+this.offset));
            return this.pos+this.offset;
        }
        else if (this.tokens[this.pos+this.offset].type!=tokenCategories.close && subSet)
        {
            consoleLog('End of subset without closing bracket at char ' + token.getDescription());
            this.error = 'End of subset without closing bracket at char ' + token.getDescription();
            return -1;
        }
   
    }

    this.testFollow = function(token,nextToken,subSet)
    {
  
        if (token.type.terminal && subSet && nextToken.type.name=='seperator')
            return true;
        var name = nextToken.type.name;
        return arrayIndexOf(token.type.follow,name) >= 0;
    }

    this.testStart = function(token)
    {
        return token.type.start;
    }
    this.testEnd = function(token)
    {
        return token.type.end;
    }
    
    this.peek = function()
    {
        if (this.pos+this.offset<this.tokens.length-1)
            return this.tokens[this.pos+this.offset+1];
        return null;
    }

    
}


function bottomUp(node)
{
    if (node.children.length==0)
        return node.value;
    var s = node.value;
    for (var i = 0; i < node.children.length; i++)
    {
        s+= bottomUp(node.children[i]);
    }
    return s;

}



function TokenNode(type,value)
{
    this.children = new Array();
    this.parent = null;
    this.value = value;
    this.type = type;
    this.block = false;
    

    this.addChild = function(c)
    {
        this.children.push(c);
    }

    this.setParent = function(p)
    {
        this.parent = p;
    }

    this.isTerminator = function()
    {
        return this.type.terminal;
    }

    this.getDescription = function()
    {
        if (this.type.name=='operator' || this.type.name=='comperator'  || this.type.name=='comperator')
        {
            return '`' + this.value.name + '`';
        }
        if (this.type.name=='open' || this.type.name=='close' || this.type.name=='seperator')
        {
            return '`' + this.type.desc + '`';

        }
        if (this.type.name=='number' || this.type.name=='string' || this.type.name=='variable' || this.type.name=='function')
        {
            return '`' + this.value + '`';
        }
        return '';
    }

}


function Token(position,type,value)
{
    this.position = position;
    this.type = type;
    this.value = value;
    this.getDescription = function()
    {
        if (this.type.name=='operator' || this.type.name=='comperator'  || this.type.name=='logical')
        { 

            return '`' + this.value.name + '`';
        }
        if (this.type.name=='open' || this.type.name=='close' || this.type.name=='seperator')
        {
            return '`' + this.type.desc + '`';

        }
        if (this.type.name=='number' || this.type.name=='string' || this.type.name=='variable' || this.type.name=='function')
        {
            return '`' + this.value + '`';
        }
        return '';
    }

    this.getType = function()
    {
        return this.type;
    }
}

function Tokenizer() 
{
    /*
     * string s
     */
    var s;

    /*
     * current position
     */
    var pos;
    var tokens;
    var operators;
    var comperators;
    var reserved;
    var reservedOpeners;
    var error;

    this.process = function(s)
    {
        this.reserved = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-_';
        this.reservedOpeners ='abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
        this.comperators = '<>=!';
        this.operators = '*/+-';
        this.logical = '&|';
        this.s = s;
        this.pos = 0;
        this.tokens = new Array();
        this.error = '';
        this.start();
    }

    this.start = function()
    {
        consoleLog('starting tokenizer');
        while (this.pos < this.s.length)
        {
            var c = this.getChar();
            if (this.isDigit(c))
            {
                var number = parseInt(c,10);
                var decimalSeperatorFound = false;
                while (this.isDigit(this.peek()))
                {
                    c = this.getChar();
                    if (c=='.' && decimalSeperatorFound)
                    {
                        this.error = 'Two decimal seperators found.';
                        return false;
                    }
                    else if (c=='.')
                    {
                        number += '.';
                        decimalSeperatorFound = true;
                    }
                    else if (decimalSeperatorFound)
                    {
                        number += c;
                    }
                    else
                    {
                        number = 10 * number + parseInt(c, 10);
                    }
                }
                if (decimalSeperatorFound)
                {
                    this.tokens.push(new Token(this.pos,tokenCategories.number,parseFloat(number)));
                }
                else
                {
                    this.tokens.push(new Token(this.pos,tokenCategories.number,number));
                }
            }
            else if (this.isOpenString(c))
            {
                var s = c;
                while (!this.isCloseString(this.peek()))
                {
                    c = this.getChar();
                    if (this.pos>=this.s.length)
                    {
                        this.error = 'Unterminated string error';
                        return;
                    }
                    if (c=='\\')
                    {
                        
                        c = this.getChar();
                        if (this.pos>=this.s.length)
                        {
                            this.error = 'Unterminated string error';
                            return;
                        }
                        if (c=='\\')
                        {
                            s+= '\\';
                        }
                        else if (c=='\'')
                        {
                            s+= '\'';
                        }
                        else
                        {
           
                            this.error = 'Unterminated backslash at ' + this.pos;
                            return;
                        }
                    }
                    else
                    {
                        s += c;
                    }
                }
                this.getChar();
                s=s.substring(1);//+=this.getChar();
                this.tokens.push(new Token(this.pos,tokenCategories.string,s));

            }
            else if (this.isOperator(c))
            {
                var type;
                if (c=='+')
                    type = tokenTypes.add;
                if (c=='-')
                    type = tokenTypes.subtract;
                if (c=='/')
                    type = tokenTypes.divide;
                if (c=='*')
                    type = tokenTypes.multiply;

                this.tokens.push(new Token(this.pos,tokenCategories.operator,type));

            }
            else if (this.isOpenBracket(c))
            {                
                this.tokens.push(new Token(this.pos,tokenCategories.open,c));
            }
            else if (this.isCloseBracket(c))
            {
                this.tokens.push(new Token(this.pos,tokenCategories.close,c));
            }
            else if (this.isSeperator(c))
            {
                this.tokens.push(new Token(this.pos,tokenCategories.seperator,c));
            }
            else if (this.isLogical(c))
            {
                var s = c;
                if (!this.isLogical(this.peek()))
                {
                   this.error = 'Single logical operator character found.';
                   return false;
                }
                c = this.getChar()
                if (c!=s)
                {
                    this.error = '& followed by | or | followed by &.';
                    return false;
                }
                if (this.isLogical(this.peek()))
                {
                   this.error = 'Three logical operator characters found.';
                   return false;
                }
                var type = tokenTypes.logand;
                if (s=='|')
                    type = tokenTypes.logor;
                this.tokens.push(new Token(this.pos,tokenCategories.logical,type));
            }
            else if (this.isComperator(c))
            {
                var s = c;
                while (this.isComperator(this.peek()))
                {
                   s += this.getChar()
                }
                var type;
                if (s=='=')
                    type = tokenTypes.eq;
                else if (s=='<')
                    type = tokenTypes.lt;
                else if (s=='<=')
                    type = tokenTypes.lte;
                else if (s=='>')
                    type = tokenTypes.gt;
                else if (s=='>=')
                    type = tokenTypes.gte;
                else if (s=='!=')
                    type = tokenTypes.ne;
                else
                {
                    this.error = 'Syntax error in comperator: ' + s;
                    return false;
                }
                this.tokens.push(new Token(this.pos,tokenCategories.comperator,type));

            }
            else if (this.isReservedOpener(c))
            {
                var s = c;
                while (this.isReserved(this.peek()))
                {
                   s += this.getChar()
                }
                this.tokens.push(new Token(this.pos,tokenCategories.variable,s));
            }
        }
    }

    this.isOpenBracket = function(c)
    {
        return c=='(';
    }

    this.isSeperator = function(c)
    {
        return c==',';
    }


    this.isCloseBracket = function(c)
    {
        return c==')';
    }

    this.isReservedOpener = function(c)
    {
        return this.reservedOpeners.indexOf(c)>=0;
    }

    this.isReserved = function(c)
    {
        return this.reserved.indexOf(c)>=0;
    }
    this.isOperator = function(c)
    {

        return this.operators.indexOf(c)>=0;
    }
    this.isLogical = function(c)
    {

        return this.logical.indexOf(c)>=0;
    }

    this.isComperator = function(c)
    {

        return this.comperators.indexOf(c)>=0;
    }

    this.isOpenString = function (c)
    {
        return c == '\'';
    }

    this.isCloseString = function (c)
    {
        return c == '\'';
    }

    this.isDigit = function(c)
    {
        if (c=='.') return true;
        return !isNaN(parseInt(c, 10));
    }

    this.peek = function(offset)
    {
        if (offset == undefined)
            offset = 0;
        
        if (this.pos < this.s.length)
        {
            var c = this.s.substring(this.pos+offset,this.pos+offset+1);
            consoleLog('Peeked ' + c + ' at position ' + this.pos);
            return c;
        }
        else
        {
            return false;
        }
    }

    this.getChar = function()
    {
        if (this.pos < this.s.length)
        {
            var c = this.s.substring(this.pos,this.pos+1);
            consoleLog('Read ' + c + ' at position ' + this.pos);
            this.pos++;
            return c;
        }
        else
        {
            return false;
        }
    }


}