public override TokenInfo ReadToken()
		{
			Token token = tokenizer.GetNext ();
			SourceLocation location = new SourceLocation(token.StartPosition,token.StartLine,token.StartColumn);
			SS.SourceSpan span = new SS.SourceSpan (ConvertToSSSrcLocation(location), ConvertToSSSrcLocation(tokenizer.Position));
			TokenTriggers trigger = GetTrigger(token.Kind);
			TokenCategory category = GetCategory (token.Kind);
			return new TokenInfo (span, category, trigger);
		}

 private static void ValidateLocations(SourceLocation start, SourceLocation end) {
     if (start.IsValid && end.IsValid) {
         if (start > end) {
             throw new ArgumentException("Start and End must be well ordered");
         }
     } else {
         if (start.IsValid || end.IsValid) {
             throw new ArgumentException("Start and End must both be valid or both invalid");
         }
     }
 }

 public HappySourceLocation(SourceUnit unit, SourceLocation start, SourceLocation end)
 {
     ContractUtils.RequiresNotNull(unit, "unit");
     ContractUtils.RequiresNotNull(start, "start");
     ContractUtils.RequiresNotNull(end, "end");
     //ContractUtils.Requires(isBefore(start, end), "start and end must be well ordered");
     if(!isBefore(start, end))
         throw new ApplicationException("start and end must be well ordered");
     this.Unit = unit;
     this.Span = new SourceSpan(start, end);
 }

		public void Init()
		{
			mockCompiler = new MockPythonCompiler();
			compiler = new DummyPythonCompilerTask(mockCompiler, @"C:\Projects\MyProject");
			compiler.TargetType = "Exe";
			compiler.OutputAssembly = "test.exe";
			
			TaskItem sourceFile = new TaskItem(@"D:\Projects\MyProject\PythonApp.Program.py");
			compiler.Sources = new ITaskItem[] {sourceFile};
			
			SourceUnit source = DefaultContext.DefaultPythonContext.CreateSourceUnit(NullTextContentProvider.Null, @"PythonApp\Program", SourceCodeKind.InteractiveCode);
			
			SourceLocation start = new SourceLocation(0, 1, 1);
			SourceLocation end = new SourceLocation(0, 2, 3);
			SourceSpan span = new SourceSpan(start, end);
			SyntaxErrorException ex = new SyntaxErrorException("Error", source, span, 1000, Severity.FatalError);
			mockCompiler.ThrowExceptionAtCompile = ex;
			
			success = compiler.Execute();
		}

 public Token Output(Symbol symbol)
 {
     var loc = new SourceLocation(index, Line, Column);
     return new Token(symbol, new SourceSpan(loc, loc));
 }

        public MSAst.Expression TransformMaybeSingleLineSuite(Statement body, SourceLocation prevStart) {
            MSAst.Expression res;
            if (body.Start.Line == prevStart.Line) {
                // avoid creating and throwing away as much line number goo as we can...
                res = body.Transform(this);
            } else {
                res = Transform(body);
            }

            MSAst.BlockExpression block = res as MSAst.BlockExpression;
            if (block != null && block.Expressions.Count > 0) {
                MSAst.DebugInfoExpression dbgInfo = block.Expressions[0] as MSAst.DebugInfoExpression;
                // body on the same line as an if, don't generate a 2nd sequence point
                if (dbgInfo != null && dbgInfo.StartLine == prevStart.Line) {
                    // we remove the debug info based upon how it's generated in DebugStatement.AddDebugInfo which is
                    // the helper method which adds the debug info.
                    if (block.Type == typeof(void)) {
                        Debug.Assert(block.Expressions.Count == 3);
                        Debug.Assert(block.Expressions[2] is MSAst.DebugInfoExpression && ((MSAst.DebugInfoExpression)block.Expressions[2]).IsClear);
                        res = block.Expressions[1];
                    } else {
                        Debug.Assert(block.Expressions.Count == 4);
                        Debug.Assert(block.Expressions[3] is MSAst.DebugInfoExpression && ((MSAst.DebugInfoExpression)block.Expressions[2]).IsClear);
                        Debug.Assert(block.Expressions[1] is MSAst.BinaryExpression && ((MSAst.BinaryExpression)block.Expressions[2]).NodeType == MSAst.ExpressionType.Assign);
                        res = ((MSAst.BinaryExpression)block.Expressions[1]).Right;
                    }
                }
            }

            if (res.Type != typeof(void)) {
                res = AstUtils.Void(res);
            }

            return res;
        }

        internal MSAst.Expression TransformLoopBody(Statement body, SourceLocation headerStart, out LabelTarget breakLabel, out LabelTarget continueLabel) {
            // Save state
            bool savedInFinally = _inFinally;
            LabelTarget savedBreakLabel = _breakLabel;
            LabelTarget savedContinueLabel = _continueLabel;

            int loopId = ++_loopOrFinallyId;
            LoopOrFinallyIds.Add(loopId, false);

            _inFinally = false;
            breakLabel = _breakLabel = Ast.Label("break");
            continueLabel = _continueLabel = Ast.Label("continue");
            MSAst.Expression result;
            try {
                result = TransformMaybeSingleLineSuite(body, headerStart);
            } finally {
                _inFinally = savedInFinally;
                _breakLabel = savedBreakLabel;
                _continueLabel = savedContinueLabel;
                LoopOrFinallyIds.Remove(loopId);
            }
            return result;
        }

        private Expression FinishSlice(Expression e0, SourceLocation start) {
            Expression e1 = null;
            Expression e2 = null;
            bool stepProvided = false;

            switch (PeekToken().Kind) {
                case TokenKind.Comma:
                case TokenKind.RightBracket:
                    break;
                case TokenKind.Colon:
                    // x[?::?]
                    stepProvided = true;
                    NextToken();
                    e2 = ParseSliceEnd();
                    break;
                default:
                    // x[?:val:?]
                    e1 = ParseExpression();
                    if (MaybeEat(TokenKind.Colon)) {
                        stepProvided = true;
                        e2 = ParseSliceEnd();
                    }
                    break;
            }
            SliceExpression ret = new SliceExpression(e0, e1, e2, stepProvided);
            ret.SetLoc(start, GetEnd());
            return ret;
        }

        Token ExpectMatch(Symbol right, Symbol left, SourceLocation leftStart)
        {
            if (Current.Symbol != right)
            {
                if (Current.Line == leftStart.Line)
                {
                    throw ReportSyntaxErrorNear("'{0}' expected",
                        right.ToTokenString());
                }

                throw ReportSyntaxErrorNear("'{0}' expected (to close '{1}' at line {2})",
                    right.ToTokenString(),
                    left.ToTokenString(),
                    leftStart.Line);
            }
            return Consume();
        }

 public void SetLoc(SourceLocation start, SourceLocation header, SourceLocation end) {
     Start = start;
     _header = header;
     End = end;
 }

		public PythonProperty(IClass declaringType, string name, SourceLocation location)
			: base(declaringType, name)
		{
			GetPropertyRegion(location);
			declaringType.Properties.Add(this);
		}

        public override void Initialize(object state, TextReader/*!*/ reader, SourceUnit sourceUnit, SourceLocation initialLocation) {
            ContractUtils.RequiresNotNull(reader, "reader");

            _sourceUnit = sourceUnit;

            _input = reader;
            _initialLocation = initialLocation;
            _currentLine = _initialLocation.Line;
            _currentLineIndex = _initialLocation.Index;
            _tokenSpan = new SourceSpan(initialLocation, initialLocation);

            SetState(LexicalState.EXPR_BEG);

            _tokenValue = new TokenValue();
            _eofReached = false;
            _unterminatedToken = false;

            DumpBeginningOfUnit();
        }

 /// <summary>
 /// Constructs a new span with a specific start and end location.
 /// </summary>
 /// <param name="start">The beginning of the span.</param>
 /// <param name="end">The end of the span.</param>
 public SourceSpan(SourceLocation start, SourceLocation end) {
     ValidateLocations(start, end);
     this._start = start;
     this._end = end;
 }

		/// <summary>
		/// Gets the body region for a class or a method.
		/// </summary>
		/// <remarks>
		/// Note that SharpDevelop line numbers are zero based but the
		/// DomRegion values are one based. IronPython columns and lines are one based.
		/// </remarks>
		/// <param name="body">The body statement.</param>
		/// <param name="header">The location of the header. This gives the end location for the
		/// method or class definition up to the colon.</param>
		public static DomRegion GetBodyRegion(Statement body, SourceLocation header)
		{
			int columnAfterColonCharacter = header.Column + 1;
			return new DomRegion(header.Line, header.Column + 1, body.End.Line, body.End.Column);
		}

 public Token OutputBuffer(Symbol symbol)
 {
     var loc1 = new SourceLocation(storedIndex, storedLine, storedColumn);
     var loc2 = new SourceLocation(index, Line, Column);
     return new Token(symbol, new SourceSpan(loc1, loc2), buffer.ToString());
 }

		void GetPropertyRegion(SourceLocation location)
		{
			int line = location.Line;
			int column = location.Column;
			Region = new DomRegion(line, column, line, column);
		}

        internal Expression/*!*/ AddDebugInfo(Expression/*!*/ expression, SourceLocation start, SourceLocation end)
        {
            //if (TotemContext.PythonOptions.GCStress != null)
            //{
            //    expression = Ast.Block(
            //        Ast.Call(
            //            typeof(GC).GetMethod("Collect", new[] { typeof(int) }),
            //            Ast.Constant(PyContext.PythonOptions.GCStress.Value)
            //        ),
            //        expression
            //    );
            //}

            return Utils.AddDebugInfo(expression, Document, start, end);
        }

 internal void ReportSyntaxError(SourceLocation start, SourceLocation end, string message, int errorCode) {
     // save the first one, the next error codes may be induced errors:
     if (_errorCode == 0) {
         _errorCode = errorCode;
     }
     _errors.Add(_sourceUnit, message, new SourceSpan(start, end), errorCode, Severity.FatalError);
 }

 internal MSAst.Expression/*!*/ AddDebugInfo(MSAst.Expression/*!*/ expression, SourceLocation start, SourceLocation end) {
     return Utils.AddDebugInfo(expression, _document, start, end);
 }

 public void AddError(string message, SourceLocation start, SourceLocation end, Severity severity, int errorCode)
 {
     _errors.Add(SourceUnit, message, new SourceSpan(start, end), errorCode, severity);
 }