public void GeneratorTest(SourceUnit sourceUnit, bool useLua52)
        {
            var options = new LuaCompilerOptions()
            {
                SkipFirstLine = true,
                UseLua52Features = useLua52,
            };

            var reader = TestUtils.OpenReaderOrIgnoreTest(sourceUnit.GetReader);

            TestUtils.AssertSyntaxError(() =>
            {
                var tokenizer = new Tokenizer(ErrorSink.Default, options);
                tokenizer.Initialize(null, reader, sourceUnit, SourceLocation.MinValue);
                var parser = new Parser(tokenizer, tokenizer.ErrorSink, options);
                var ast = parser.Parse();
                Assert.That(ast, Is.Not.Null);

                var codeContext = new CodeContext((LuaContext)sourceUnit.LanguageContext);

                var gen = new Generator(codeContext);
                var expr = gen.Compile(ast, sourceUnit);
                Assert.That(expr, Is.Not.Null);
            });
        }

        public OptimizedScriptCode(Scope optimizedScope, DlrMainCallTarget optimizedTarget, SourceUnit sourceUnit)
            : base(sourceUnit) {
            ContractUtils.RequiresNotNull(optimizedScope, "optimizedScope");

            _optimizedScope = optimizedScope;
            _optimizedTarget = optimizedTarget;
        }

        internal SourceUnitReader(SourceUnit sourceUnit, TextReader textReader)
        {
            Assert.NotNull(sourceUnit, textReader);

            _textReader = textReader;
            _sourceUnit = sourceUnit;
        }

        internal Lexer(SourceUnit unit, HappyLanguageContext languageContext)
        {
            _reader = new InputReader(unit);
            _errorCollector = new ErrorCollector(languageContext.ErrorSink);
            PushState(LexerMode.LexingStatement);

            _keywords["if"] = HappyTokenKind.KeywordIf;
            _keywords["else"] = HappyTokenKind.KeywordElse;
            _keywords["while"] = HappyTokenKind.KeywordWhile;
            _keywords["for"] = HappyTokenKind.KeywordFor;
            _keywords["in"] = HappyTokenKind.KeywordIn;
            _keywords["between"] = HappyTokenKind.KeywordBetween;
            _keywords["where"] = HappyTokenKind.KeywordWhere;
            _keywords["lookup"] = HappyTokenKind.KeywordLookup;
            _keywords["default"] = HappyTokenKind.KeywordDefault;
            _keywords["true"] = HappyTokenKind.LiteralBool;
            _keywords["false"] = HappyTokenKind.LiteralBool;
            _keywords["return"] = HappyTokenKind.KeywordReturn;
            _keywords["def"] = HappyTokenKind.KeywordDef;
            _keywords["null"] = HappyTokenKind.LiteralNull;
            _keywords["load"] = HappyTokenKind.KeywordLoad;
            _keywords["use"] = HappyTokenKind.KeywordUse;
            _keywords["new"] = HappyTokenKind.KeywordNew;
            _keywords["function"] = HappyTokenKind.KeywordFunction;
            _keywords["break"] = HappyTokenKind.KeywordBreak;
            _keywords["continue"] = HappyTokenKind.KeywordContinue;
            _keywords["switch"] = HappyTokenKind.KeywordSwitch;
            _keywords["case"] = HappyTokenKind.KeywordCase;
        }

 HappySourceLocation(SourceUnit unit, SourceSpan span)
 {
     //ContractUtils.RequiresNotNull(unit, "unit");
     ContractUtils.RequiresNotNull(span, "span");
     this.Unit = unit;
     this.Span = span;
 }

		public override void Initialize(object state, TextReader textreader, SourceUnit sourceUnit, SS.SourceLocation initialLocation)
		{
			tokenizer = new Tokenizer (textreader.ReadToEnd ().ToCharArray (), new IdentifierTable ());
			tokenizer.Position = ConvertToMJCSrcLocation(initialLocation);
			this.sourceUnit = sourceUnit;
			this.state = state;
		}

        public LambdaFunctionExpr(SourceUnit/*!*/ sourceUnit,
            Position position, Position entireDeclarationPosition, ShortPosition headingEndPosition, ShortPosition declarationBodyPosition,
            Scope scope, NamespaceDecl ns,
            bool aliasReturn, List<FormalParam>/*!*/ formalParams, List<FormalParam> useParams,
            List<Statement>/*!*/ body)
            : base(position)
        {
            Debug.Assert(formalParams != null && body != null);

            // inject use parameters at the begining of formal parameters
            if (useParams != null && useParams.Count > 0)
            {
                if (formalParams.Count == 0)
                    formalParams = useParams;   // also we don't want to modify Parser.emptyFormalParamListIndex singleton.
                else
                    formalParams.InsertRange(0, useParams);
            }
            
            //this.ns = ns;
            this.signature = new Signature(aliasReturn, formalParams);
            this.useParams = useParams;
            //this.typeSignature = new TypeSignature(genericParams);
            //this.attributes = new CustomAttributes(attributes);
            this.body = body;
            this.entireDeclarationPosition = entireDeclarationPosition;
            this.headingEndPosition = headingEndPosition;
            this.declarationBodyPosition = declarationBodyPosition;

            //QualifiedName qn = (ns != null) ? new QualifiedName(this.name, ns.QualifiedName) : new QualifiedName(this.name);
            //function = new PhpFunction(qn, memberAttributes, signature, typeSignature, isConditional, scope, sourceUnit, position);
            function = new PhpLambdaFunction(this.signature, sourceUnit, position);
            function.WriteUp(new TypeSignature(FormalTypeParam.EmptyList).ToPhpRoutineSignature(function));
        }

            public override void Add(SourceUnit sourceUnit, string message, SourceSpan span, int errorCode, Severity severity)
            {
                base.Add(sourceUnit, message, span, errorCode, severity);

                Console.Error.WriteLine("{0}({1}:{2}): {3}: RB{4}: {5}", sourceUnit.Name, span.Start.Line, span.Start.Column,
                    severity, errorCode, message);
            }

 public override void Add(SourceUnit sourceUnit, string message, SourceSpan span, int errorCode, Severity severity) {
     if (severity == Severity.Warning) {
         PythonOps.SyntaxWarning(message, sourceUnit, span, errorCode);
     } else {
         throw PythonOps.SyntaxError(message, sourceUnit, span, errorCode);
     }
 }

		internal override Evaluation EvaluatePriorAnalysis(SourceUnit/*!*/ sourceUnit)
		{
			Evaluation left_eval = leftExpr.EvaluatePriorAnalysis(sourceUnit);
			Evaluation right_eval = leftExpr.EvaluatePriorAnalysis(sourceUnit);

			return Evaluation.ReadOnlyEvaluate(this, left_eval, right_eval);
		}

 public LuaScriptCode(CodeContext context, SourceUnit sourceUnit, Expression<Func<IDynamicMetaObjectProvider, dynamic>> chunk)
     : base(sourceUnit)
 {
     Contract.Requires(chunk != null);
     Context = context;
     _exprLambda = chunk;
 }

        public override ScriptCode CompileSourceCode(SourceUnit sourceUnit, CompilerOptions options, ErrorSink errorSink)
        {
            try
            {
                switch (sourceUnit.Kind)
                {
                    case SourceCodeKind.SingleStatement:
                    case SourceCodeKind.Expression:
                    case SourceCodeKind.AutoDetect:
                    case SourceCodeKind.InteractiveCode:
                        return new TotemScriptCode(
                            engine, engine.ParseExprToLambda(sourceUnit),
                            sourceUnit);

                    //case SourceCodeKind.Statements:
                    //case SourceCodeKind.File:
                    //    return new TotemScriptCode(
                    //        engine, engine.ParseFileToLambda(sourceUnit.Path, sourceUnit.GetCode()),
                    //        sourceUnit);

                    default:
                        throw Assert.Unreachable;
                }
            }
            catch (Exception e)
            {
                // Real language implementation would have a specific type
                // of exception.  Also, they would pass errorSink down into
                // the parser and add messages while doing tighter error
                // recovery and continuing to parse.
                errorSink.Add(sourceUnit, e.Message, SourceSpan.None, 0,
                              Severity.FatalError);
                return null;
            }
        }

        public override void Add(SourceUnit sourceUnit, string message, SourceSpan span, int errorCode, Severity severity) {
            if (severity == Severity.Warning && !ReportWarning(_context.Verbose, errorCode)) {
                return;
            }

            CountError(severity);

            string path;
            string codeLine;
            int line = span.Start.Line;
            if (sourceUnit != null) {
                path = sourceUnit.Path;
                codeLine = (line > 0) ? sourceUnit.GetCodeLine(line) : null;
            } else {
                path = null;
                codeLine = null;
            }

            if (severity == Severity.Error || severity == Severity.FatalError) {
                throw new SyntaxError(message, path, line, span.Start.Column, codeLine);
            } else {

                if (_WriteSite == null) {
                    Interlocked.CompareExchange(
                        ref _WriteSite,
                        CallSite<Func<CallSite, object, object, object>>.Create(RubyCallAction.Make(_context, "write", 1)),
                        null
                    );
                }

                message = RubyContext.FormatErrorMessage(message, "warning", path, line, span.Start.Column, null);

                _WriteSite.Target(_WriteSite, _context.StandardErrorOutput, MutableString.CreateMutable(message));
            }
        }

 /// <summary>
 /// Creates a wrapper around a function which represents entry into a chunk of code
 /// </summary>
 public ExecutionContextExpression(CodeContext context, Expression scopeVariable, SourceUnit source, Expression body)
 {
     _context = context;
     _body = body;
     _scope = scopeVariable;
     _source = source;
 }

        private SourceUnitTree Parse(SourceUnit sourceUnit, bool allowSingle, out bool isExpression)
        {
            isExpression = false;
            IronyParser parser = new IronyParser(allowSingle ? singleStatement : fullGrammar);
            parser.Context.Mode = ParseMode.CommandLine;

            scopes = new Stack<LexicalScopeBuilder>();
            EnterTopLevelScope();
            try
            {
                var parsedScript = parser.Parse(sourceUnit.GetCode());
                if (parsedScript.HasErrors())
                {
                    sourceUnit.CodeProperties = ScriptCodeParseResult.Invalid;
                    return null;
                }

                if (sourceUnit.Kind == SourceCodeKind.InteractiveCode && parser.Context.Status == ParserStatus.AcceptedPartial)
                {
                    sourceUnit.CodeProperties = ScriptCodeParseResult.IncompleteStatement;
                    return null;
                }

                sourceUnit.CodeProperties = ScriptCodeParseResult.Complete;
                return BuildSourceTree(parsedScript.Root, sourceUnit, allowSingle, out isExpression);
            }
            catch (Exception e)
            {
                throw;
            }
            finally
            {
                LeaveScope();
            }
        }

        public HappyLambdaScriptCode Analyze(Module module, SourceUnit sourceUnit)
        {
            Dictionary<string, HappyNamespaceTracker> rootNamespaces = LoadAllAssemblies(module.LoadDirectives);

            AstVisitorBase[] visitors =
                {
                    new BinaryExpressionFixerVisitor(),
                    new BuildSymbolTablesVisitor(this, _errorCollector, rootNamespaces),
                    new ResolveSymbolsVisitor(_errorCollector),
                    new SemanticVisitor(_errorCollector)
                };

            foreach (var v in visitors)
                module.Accept(v);

            prepareAssemblyGenerator();

            module.Accept(this);

            Expression expression = _expressionStack.Pop();
            DebugAssert.IsZero(_expressionStack.Count, "AstAnalyzer didn't consume all expressions on the stack");

            var runtimeContextInitializer = (LambdaExpression)expression;
            return new HappyLambdaScriptCode(sourceUnit, compileDynamicAssembly(runtimeContextInitializer));
        }

 public static Module ParseModule(SourceUnit sourceUnit, HappyLanguageContext languageContext)
 {
     Lexer lexer = new Lexer(sourceUnit, languageContext);
     Parser parser = new Parser(lexer, languageContext);
     Module retval = null;
     try
     {
         retval = parser.Parse();
     }
     catch(AbortParseException)
     {
         //Left blank intentionally because AbortParseExceptions should never
         //be allowed to bubble up to the caller.
     }
     catch(SyntaxErrorException)
     {
         //We actually want to allow this exception to propogate as-is back
         //to the caller.  This is needed because of the catch-all below
         throw;
     }
     catch(Exception e)
     {
         //All other exceptions are to be considered an internal error
         //We may not know the exact location of the error, but we'll just
         //take the next token out of the lexer to give *some* kind of
         //idea line might contain the error
         DebugAssert.IsNotNull(e);
         Token t = lexer.NextToken();
         throw new InternalSourceException(e, t.Location,
             "The parser threw an unhandled exception.  This is usually caused by a syntax error " +
             "in the script being parsed.  The error ocurred at or before the this locaiton.");
     }
     return retval;
 }

        public LambdaFunctionExpr(SourceUnit/*!*/ sourceUnit,
            Text.Span span, Text.Span entireDeclarationPosition, int headingEndPosition, int declarationBodyPosition,
            Scope scope, NamespaceDecl ns,
            bool aliasReturn, List<FormalParam>/*!*/ formalParams, List<FormalParam> useParams,
            List<Statement>/*!*/ body)
            : base(span)
        {
            Debug.Assert(formalParams != null && body != null);

            // inject use parameters at the begining of formal parameters
            if (useParams != null && useParams.Count > 0)
            {
                if (formalParams.Count == 0)
                    formalParams = useParams;   // also we don't want to modify Parser.emptyFormalParamListIndex singleton.
                else
                    formalParams.InsertRange(0, useParams);
            }
            
            //this.ns = ns;
            this.signature = new Signature(aliasReturn, formalParams);
            this.useParams = useParams;
            //this.typeSignature = new TypeSignature(genericParams);
            //this.attributes = new CustomAttributes(attributes);
            this.body = body;
            this.entireDeclarationPosition = entireDeclarationPosition;
            this.headingEndPosition = headingEndPosition;
            this.declarationBodyPosition = declarationBodyPosition;
        }

        internal LightExpression<Func<TotemEngine, IDynamicMetaObjectProvider, object>> ParseExprToLambda(SourceUnit sourceUnit)
        {
            bool isExpression;
            var ast = new Parser().ParseExpression(sourceUnit, out isExpression);
            if (ast == null)
                return null;
            var scope = new AnalysisScope.TopLevel(
                this,
                Expression.Parameter(typeof(TotemEngine), "totemRuntime"),
                Expression.Parameter(typeof(IDynamicMetaObjectProvider), "global"));

            var ret = Expression.Variable(typeof(object), "#__returnVariable");
            List<Expression> body = new List<Expression>();
            var b = AstGenerator.Generate(ast, scope);
            body.Add(Expression.Assign(ret, Expression.Convert(b, typeof(object))));
            if (sourceUnit.Kind == SourceCodeKind.InteractiveCode)
            {
                if(isExpression) body.Add(Expression.Call(typeof(Console).GetMethod("WriteLine", new[] { typeof(object) }), ret));
                body.Add(Expression.Call(typeof(Console).GetMethod("WriteLine", Type.EmptyTypes)));
                body.Add(ret);
            }

            var moduleFun = Utils.LightLambda<Func<TotemEngine, IDynamicMetaObjectProvider, object>>(
                typeof(object),
                Expression.Block(new ParameterExpression[] { ret }, body),
                "_snippet_",
                new List<ParameterExpression>() { scope.RuntimeExpr, scope.GlobalExpr }
            );
            //var moduleFun = Expression.Lambda<Func<TotemEngine, IDynamicMetaObjectProvider, object>>(
            //    Expression.Block(new ParameterExpression[] { ret }, body),
            //    scope.RuntimeExpr,
            //    scope.ModuleExpr
            //);
            return moduleFun;
        }

 internal RubyScriptCode(ScriptCodeFunc/*!*/ target, SourceUnit/*!*/ sourceUnit, TopScopeFactoryKind kind)
     : base(sourceUnit)
 {
     Assert.NotNull(target);
     _target = target;
     _kind = kind;
 }