def analyzeJSCodesFinerBlock(script, display=False):
  try:
    t1 = time.time()
    parser = Parser()
    script = script.strip()
    if script.startswith('<!--') and script.endswith('-->'):
      script = script[4: -3]
    tree = parser.parse(script)
    visitor = MyVisitor( display)
    visitor.visit(tree, 0)
    if len(visitor.first_level_seq) != len(visitor.scripts):
      print >>sys.stderr, "error parsing script: scripts and seqs length inconsistent "+script[:100]
      return None, None
    t2 = time.time()
    total_time = t2 - t1
    total_len = float(len(script))
    try:
      portion = [len(x)/total_len for x in visitor.scripts]
      for i in range(len(portion)):
        t = total_time * portion[i]
        print "AST_TIME: %f %d" %(t, len(visitor.scripts[i]))    
    except:
      pass
    return visitor.first_level_seq, visitor.scripts
  except Exception as e:
    print >>sys.stderr, "error parsing script: "+str(e)+" || [START]"+script[:100]+"[END]"
    return None, None

 def find_first_match(test_file_content, pos):
   test_file_content = TestMethodMatcher.UnitTest.insert_mark(test_file_content, pos)
   if not test_file_content:
     return None
   parser = Parser()
   tree = parser.parse(test_file_content)
   return TestMethodMatcher.UnitTest.get_test_name(tree)

def extract_glow_lib():
    runjs = norm_path('untrusted/run.js')
    parser = JSParser()

    with open(runjs) as f:
        tree = parser.parse(f.read())

    for node in nodevisitor.visit(tree):
        if (isinstance(node, ast.Assign) and
            isinstance(node.left, ast.DotAccessor) and
            node.left.identifier.value == 'glowscript_libraries' and
            isinstance(node.right, ast.Object)):
                break
    else:
        print('Parsing {} failed'.format(runjs))
        exit(-1)

    return preproc_lib_path({
        prop.left.value:
            [
                eval(lib.value)
                for lib in prop.right.items
                if isinstance(lib, ast.String)
            ]
            for prop in node.right.properties
    })

    def extract(self, node):
        '''
        Given a BeautifulSoup representation of an HTML document,
        return a list of all code snippets in that document.
        '''

        node_code = None
        code_snippets = []

        # Attempt to parse content for a code node as JavaScript.
        # Mark the content as a code snippet if it is parsed successfully.
        # Skip nodes with nothing but whitespace content.
        if type(node) is Tag and node.name in self.TAGS:
            if node.text.strip() != '':
                try:
                    js_parser = JavaScriptParser()
                    js_parser.parse(node.text)
                except (SyntaxError, TypeError, AttributeError):
                    logger.debug("Code content could not be parsed as JavaScript.")
                else:
                    node_code = node.text
                    code_snippets.append(node_code)

        # If this node did not contain valid code, then visit all children
        # and check them for code.
        if node_code is None and hasattr(node, 'children'):
            for child_element in node.children:
                code_snippets.extend(self.extract(child_element))

        return code_snippets

def load_data():
    # Read list names (toplists.js)
    f = open("./data/masterbugtable.js", "r")
    tmp = f.read()
    f.close()
    masterbugtable = json.loads(tmp[tmp.index("{") :])
    # find and process latest test data
    f = open("./data/testing/index.json", "r")
    tmp = f.read()
    f.close()
    test_file_index = json.loads(tmp)
    f = open("./data/testing/%s" % test_file_index[-1], "r")
    test_data = {}
    test_reader = csv.reader(f)
    for line in test_reader:
        test_data[str(line[0])] = {"bug": line[0], "test_date": line[1], "ua": line[2], "test_state": line[3]}
    f.close()
    f = open("./data/sitedata.js", "r")
    parser = Parser()
    tree = parser.parse(f.read())
    f.close()
    return {
        "masterbugtable": masterbugtable,
        "test_result_files": test_file_index,
        "test_results": test_data,
        "tests_parsed": tree,
    }

def minify(text, mangle=False, mangle_toplevel=False):
    parser = Parser()
    tree = parser.parse(text)
    if mangle:
        mangler.mangle(tree, toplevel=mangle_toplevel)
    minified = ECMAMinifier().visit(tree)
    return minified

 def _parse_succeeds(self, text):
     try:
         parser = JsParser()
         parser.parse(text)
     except (SyntaxError, TypeError):
         return False
     else:
         return True

    def case(self, case):
        parser_a = Parser()
        result_a = parser_a.parse(case)

        parser_b = Parser()
        result_b = parser_b.parse(case)

        self.assertEqual(result_a, result_b)

 def test_func(self):
     parser = Parser()
     tree = parser.parse(input)
     mangle(tree, toplevel=True)
     self.assertMultiLineEqual(
         textwrap.dedent(tree.to_ecma()).strip(),
         textwrap.dedent(expected).strip()
         )

def whileExtract(s):
    '''Extracts all the while loops in the script. '''
    l = []
    parser = Parser()
    tree = parser.parse(s)
    for node in nodevisitor.visit(tree):
        if isinstance(node, ast.While):
            l+=[node.to_ecma()]
    return l

def _parse_redirect_to_security_challenge_script(script: str) -> str:
    """ Parses the script which redirects us to security challenge page and gets that URL. """
    parser = Parser()
    tree = parser.parse(script)
    nodes = [node for node in nodevisitor.visit(tree) if isinstance(node, ast.Assign)]
    for node in nodevisitor.visit(tree):
        if isinstance(node, ast.Assign) and hasattr(node, 'left') and isinstance(node.left, ast.DotAccessor):
            children = node.left.children()
            if len(children) == 2 and children[0].value == 'window' and children[1].value == 'location':
                return node.right.value.strip('\'"')

 def test_bug_no_semicolon_at_the_end_of_block_plus_newline_at_eof(self):
     # https://github.com/rspivak/slimit/issues/3
     text = textwrap.dedent("""
     function add(x, y) {
       return x + y;
     }
     """)
     parser = Parser()
     tree = parser.parse(text)
     self.assertTrue(bool(tree.children()))

def analyzeJSCodes(script, display=False):
  try:
    parser = Parser()
    tree = parser.parse(script)
    visitor = MyVisitor( display)
    visitor.visit(tree, 0)
    #print "first_level_seq: %d" %len(visitor.first_level_seq)
    return visitor.node_order_list
  except Exception as e:
    print >>sys.stderr, "error parsing script: "+str(e)+" || "+script
    return None

 def _test_function_expression(self):
     text = """
     if (true) {
       function() {
         foo;
         location = 'http://anywhere.com';
       }
     }
     """
     parser = Parser()
     parser.parse(text)

def rewriteJSCodes(script, display=False):
  try:
    parser = Parser()
    tree = parser.parse(script)
    visitor = RewriteVisitor( display)
    visitor.visit(tree, 0)
    x = ECMAVisitor().visit(tree)
    print x
    #print tree.to_ecma()
    #print "first_level_seq: %d" %len(visitor.first_level_seq)
  except Exception as e:
    print >>sys.stderr, "error parsing script: "+str(e)+" || "+script

def removeDeclarations(js_file):
  parser = Parser()
  tree = parser.parse(js_file)
  output = ""
  for child in tree.children():
    if type(child) != ast.VarStatement:
      output += (child.to_ecma() + "\n")
    else:
      nodes = [x for x in nodevisitor.visit(child)]
      if type(nodes[0].initializer) not in [ast.String, ast.Number, ast.BinOp]:
        output += (child.to_ecma() + "\n")
  return output

def main():
    args = parse_args()

    with open(args.filename) as f:
        source = f.read()

    parser = Parser()
    tree = parser.parse(source)

    visitor = ConstantReductionVisitor(args.debug)
    tree = visitor.visit(tree)
    print tree.to_ecma()

def episode():
    url = h.extract_var(args, 'url')

    name = h.extract_var(args, 'name')

    soup = BeautifulSoup(h.make_request(url, cookie_file, cookie_jar))

    div = h.bs_find_with_class(soup, 'div', 'video-player')

    scripts = div.findAll('script')
    script = None
    for _script in scripts:
        if 'CryptoJS' in _script.text:
            script = _script
            break

    url = ''
    if script:
        script_text = script.text
        _dailytoday = ''
        _subject = ''

        parser = Parser()
        tree = parser.parse(script.text)
        for node in tree.children():
            ecma = node.to_ecma()
            if ecma.startswith('var dailytoday ='):
                _dailytoday = node.children()[0].children()[1].to_ecma()[1:-1]
            elif ecma.startswith('var subject ='):
                _subject = node.children()[0].children()[1].to_ecma()[1:-1]
            # elif "var bigmumbai = " not in ecma and "bigmumbai = " in ecma:
            #     print ecma

        if _dailytoday and _subject:
            url = decrypt.decrypt_url(_dailytoday, _subject)
        else:
            url = script.text.split('bigmumbai = ', 2)[2].split(';')[0][1:-1]

        print url
        plot = h.bs_find_with_class(soup, 'div', 'vp-info').find('span', {'itemprop': 'description'}).text
        thumbnail = soup.find('div', {'itemprop': 'video'}).find('meta', {'itemprop': 'thumbnailUrl'})['content']
        h.add_dir_video(addon_handle, name, url, thumbnail, plot)
    else:
        iframe = div.find('iframe')
        if iframe:
            attrs = dict(iframe.attrs)
            youtube_url = attrs['src']
            print youtube_url
            video_id = urlparse.urlparse(youtube_url).path.replace('/embed/', '')
            url = 'plugin://plugin.video.youtube/play/?video_id=%s' % video_id
            h.add_dir_video(addon_handle, name, url, '', '')

def ParsingOfFunction():
    index = 0
    parser = Parser()
    tree = parser.parse('l = 0;l = h;')

    # for node in nodevisitor.visit(tree):
    #   if isinstance(node, ast.Identifier) and node.value == 'i':
    #      node.value = 'hello'

    x = tree.to_ecma()  # print awesome javascript :)
    # print x;

    # print "Opening the file..."
    # target = open("file.txt",'w')
    # target.write(x)
    # target.close()
    lines = [line.rstrip('\n') for line in open('file.txt')]

    map = {}
    temp = ""
    i = 0

    # print lines;

    for str in lines:

        if ';' in str:
            temp = temp + str
            temp = temp.lstrip(";")
            # print temp;
            map.__setitem__(i, temp)

            print "Going into the lexer function --------------"
            lex = LexingofFunction(temp)
            temp = ""
            i += 1

        else:

            temp = temp + str

    # print map;

    print "----------------------------- print heap now"

    for x in heap:
        print x.name
        print x.level
    return

def treeWalker(js_file):
  parser = Parser()
  tree = parser.parse(js_file)
  variables = {}
  for child in tree.children():
    if type(child) == ast.VarStatement:
      try:
        nodes = [x for x in nodevisitor.visit(child)]
        if   type(nodes[0].initializer) == ast.String:
          variables[nodes[0].identifier.value] = nodes[0].initializer.value
        elif type(nodes[0].initializer) == ast.Number:
          variables[nodes[0].identifier.value] = eval(nodes[0].initializer.to_ecma())
        elif type(nodes[0].initializer) == ast.BinOp:
          variables[nodes[0].identifier.value] = eval(nodes[0].initializer.to_ecma())
        else:
          print((nodes[0].identifier.value, nodes[0].initializer))
      except Exception as e:
        print (child.to_ecma())
  return variables