def try_subproc_toks(self, node, strip_expr=False):
     """Tries to parse the line of the node as a subprocess."""
     line, nlogical, idx = get_logical_line(self.lines, node.lineno - 1)
     if self.mode == "eval":
         mincol = len(line) - len(line.lstrip())
         maxcol = None
     else:
         mincol = max(min_col(node) - 1, 0)
         maxcol = max_col(node)
         if mincol == maxcol:
             maxcol = find_next_break(line, mincol=mincol, lexer=self.parser.lexer)
         elif nlogical > 1:
             maxcol = None
         elif maxcol < len(line) and line[maxcol] == ";":
             pass
         else:
             maxcol += 1
     spline = subproc_toks(
         line,
         mincol=mincol,
         maxcol=maxcol,
         returnline=False,
         lexer=self.parser.lexer,
     )
     if spline is None or spline != "![{}]".format(line[mincol:maxcol].strip()):
         # failed to get something consistent, try greedy wrap
         spline = subproc_toks(
             line,
             mincol=mincol,
             maxcol=maxcol,
             returnline=False,
             lexer=self.parser.lexer,
             greedy=True,
         )
     if spline is None:
         return node
     try:
         newnode = self.parser.parse(
             spline,
             mode=self.mode,
             filename=self.filename,
             debug_level=(self.debug_level > 2),
         )
         newnode = newnode.body
         if not isinstance(newnode, AST):
             # take the first (and only) Expr
             newnode = newnode[0]
         increment_lineno(newnode, n=node.lineno - 1)
         newnode.col_offset = node.col_offset
         if self.debug_level > 1:
             msg = "{0}:{1}:{2}{3} - {4}\n" "{0}:{1}:{2}{3} + {5}"
             mstr = "" if maxcol is None else ":" + str(maxcol)
             msg = msg.format(self.filename, node.lineno, mincol, mstr, line, spline)
             print(msg, file=sys.stderr)
     except SyntaxError:
         newnode = node
     if strip_expr and isinstance(newnode, Expr):
         newnode = newnode.value
     return newnode

 def try_subproc_toks(self, node, strip_expr=False):
     """Tries to parse the line of the node as a subprocess."""
     line, nlogical, idx = get_logical_line(self.lines, node.lineno - 1)
     if self.mode == 'eval':
         mincol = len(line) - len(line.lstrip())
         maxcol = None
     else:
         mincol = max(min_col(node) - 1, 0)
         maxcol = max_col(node)
         if mincol == maxcol:
             maxcol = find_next_break(line, mincol=mincol,
                                      lexer=self.parser.lexer)
         elif nlogical > 1:
             maxcol = None
         elif maxcol < len(line) and line[maxcol] == ';':
             pass
         else:
             maxcol += 1
     spline = subproc_toks(line, mincol=mincol, maxcol=maxcol,
                           returnline=False, lexer=self.parser.lexer)
     if spline is None or len(spline) < len(line[mincol:maxcol]) + 2:
         # failed to get something consistent, try greedy wrap
         # The +2 comes from "![]" being length 3, minus 1 since maxcol
         # is one beyond the total length for slicing
         spline = subproc_toks(line, mincol=mincol, maxcol=maxcol,
                               returnline=False, lexer=self.parser.lexer,
                               greedy=True)
     if spline is None:
         return node
     try:
         newnode = self.parser.parse(spline, mode=self.mode,
                                     filename=self.filename,
                                     debug_level=(self.debug_level > 2))
         newnode = newnode.body
         if not isinstance(newnode, AST):
             # take the first (and only) Expr
             newnode = newnode[0]
         increment_lineno(newnode, n=node.lineno - 1)
         newnode.col_offset = node.col_offset
         if self.debug_level > 1:
             msg = ('{0}:{1}:{2}{3} - {4}\n'
                    '{0}:{1}:{2}{3} + {5}')
             mstr = '' if maxcol is None else ':' + str(maxcol)
             msg = msg.format(self.filename, node.lineno,
                              mincol, mstr, line, spline)
             print(msg, file=sys.stderr)
     except SyntaxError:
         newnode = node
     if strip_expr and isinstance(newnode, Expr):
         newnode = newnode.value
     return newnode

 def try_subproc_toks(self, node, strip_expr=False):
     """Tries to parse the line of the node as a subprocess."""
     line = self.lines[node.lineno - 1]
     if self.mode == 'eval':
         mincol = len(line) - len(line.lstrip())
         maxcol = None
     else:
         mincol = min_col(node)
         maxcol = max_col(node)
         if mincol == maxcol:
             maxcol = find_next_break(line, mincol=mincol,
                                      lexer=self.parser.lexer)
         else:
             maxcol += 1
     spline = subproc_toks(line,
                           mincol=mincol,
                           maxcol=maxcol,
                           returnline=False,
                           lexer=self.parser.lexer)
     if spline is None:
         return node
     try:
         newnode = self.parser.parse(spline, mode=self.mode)
         newnode = newnode.body
         if not isinstance(newnode, AST):
             # take the first (and only) Expr
             newnode = newnode[0]
         increment_lineno(newnode, n=node.lineno - 1)
         newnode.col_offset = node.col_offset
     except SyntaxError:
         newnode = node
     if strip_expr and isinstance(newnode, Expr):
         newnode = newnode.value
     return newnode

def test_subproc_toks_hello_mom_second():
    fst = "echo 'hello'"
    sec = "echo 'mom'"
    s = "{0}; {1}".format(fst, sec)
    exp = "{0}; ![{1}]".format(fst, sec)
    obs = subproc_toks(s, lexer=LEXER, mincol=len(fst), returnline=True)
    assert exp == obs

def test_subproc_toks_hello_mom_first():
    fst = "echo 'hello'"
    sec = "echo 'mom'"
    s = "{0}; {1}".format(fst, sec)
    exp = "![{0}]; {1}".format(fst, sec)
    obs = subproc_toks(s, lexer=LEXER, maxcol=len(fst) + 1, returnline=True)
    assert exp == obs

def test_subproc_toks_ls_l_semi_ls_second():
    lsdl = "ls -l"
    ls = "ls"
    s = "{0}; {1}".format(lsdl, ls)
    exp = "{0}; ![{1}]".format(lsdl, ls)
    obs = subproc_toks(s, lexer=LEXER, mincol=7, returnline=True)
    assert exp == obs

def test_subproc_toks_ls_l_semi_ls_first():
    lsdl = "ls -l"
    ls = "ls"
    s = "{0}; {1}".format(lsdl, ls)
    exp = "![{0}]; {1}".format(lsdl, ls)
    obs = subproc_toks(s, lexer=LEXER, maxcol=6, returnline=True)
    assert exp == obs

def test_subproc_toks_indent_ls_str():
    ind = "    "
    s = 'ls "wakka"'
    com = "  # lets list"
    exp = "{0}![{1}]{2}".format(ind, s, com)
    obs = subproc_toks(ind + s + com, lexer=LEXER, returnline=True)
    assert exp == obs

def test_subproc_toks_indent_ls_nl():
    s = "ls -l"
    exp = INDENT + "![{0}]\n".format(s)
    obs = subproc_toks(
        INDENT + s + "\n", mincol=len(INDENT), lexer=LEXER, returnline=True
    )
    assert exp == obs

def test_subproc_toks_indent_ls_comment():
    ind = '    '
    s = 'ls -l'
    com = '  # lets list'
    exp = '{0}![{1}]{2}'.format(ind, s, com)
    obs = subproc_toks(ind + s + com, lexer=LEXER, returnline=True)
    assert (exp == obs)

def test_subproc_hello_mom_second():
    fst = "echo 'hello'"
    sec = "echo 'mom'"
    s = '{0}; {1}'.format(fst, sec)
    exp = '{0}; $[{1}]'.format(fst, sec)
    obs = subproc_toks(s, lexer=LEXER, mincol=len(fst), returnline=True)
    assert_equal(exp, obs)

def test_subproc_hello_mom_first():
    fst = "echo 'hello'"
    sec = "echo 'mom'"
    s = '{0}; {1}'.format(fst, sec)
    exp = '$[{0}]; {1}'.format(fst, sec)
    obs = subproc_toks(s, lexer=LEXER, maxcol=len(fst)+1, returnline=True)
    assert_equal(exp, obs)

 def try_subproc_toks(self, node):
     """Tries to parse the line of the node as a subprocess."""
     line = self.lines[node.lineno - 1]
     if self.mode == 'eval':
         mincol = len(line) - len(line.lstrip())
         maxcol = None
     else: 
         mincol = min_col(node)
         maxcol = max_col(node) + 1
     spline = subproc_toks(line,
                           mincol=mincol,
                           maxcol=maxcol,
                           returnline=False,
                           lexer=self.parser.lexer)
     try:
         newnode = self.parser.parse(spline, mode=self.mode)
         newnode = newnode.body
         if not isinstance(newnode, AST):
             # take the first (and only) Expr
             newnode = newnode[0]
         newnode.lineno = node.lineno
         newnode.col_offset = node.col_offset
     except SyntaxError:
         newnode = node
     return newnode

def test_subproc_toks_ls_l_semi_ls_first():
    lsdl = 'ls -l'
    ls = 'ls'
    s = '{0}; {1}'.format(lsdl, ls)
    exp = '$[{0}]; {1}'.format(lsdl, ls)
    obs = subproc_toks(s, lexer=LEXER, maxcol=6, returnline=True)
    assert_equal(exp, obs)

def test_subproc_toks_ls_l_semi_ls_second():
    lsdl = 'ls -l'
    ls = 'ls'
    s = '{0}; {1}'.format(lsdl, ls)
    exp = '{0}; $[{1}]'.format(lsdl, ls)
    obs = subproc_toks(s, lexer=LEXER, mincol=7, returnline=True)
    assert_equal(exp, obs)

def test_subproc_toks_indent_ls_str():
    ind = '    '
    s = 'ls "wakka"'
    com = '  # lets list'
    exp = '{0}$[{1}]{2}'.format(ind, s, com)
    obs = subproc_toks(ind + s + com, lexer=LEXER, returnline=True)
    assert_equal(exp, obs)

 def _parse_ctx_free(self, input, mode='exec'):
     last_error_line = last_error_col = -1
     parsed = False
     original_error = None
     while not parsed:
         try:
             tree = self.parser.parse(input,
                                      filename=self.filename,
                                      mode=mode,
                                      debug_level=self.debug_level)
             parsed = True
         except IndentationError as e:
             if original_error is None:
                 raise e
             else:
                 raise original_error
         except SyntaxError as e:
             if original_error is None:
                 original_error = e
             if (e.loc is None) or (last_error_line == e.loc.lineno and
                                    last_error_col in (e.loc.column + 1,
                                                       e.loc.column)):
                 raise original_error
             last_error_col = e.loc.column
             last_error_line = e.loc.lineno
             idx = last_error_line - 1
             lines = input.splitlines()
             line = lines[idx]
             if input.endswith('\n'):
                 lines.append('')
             if len(line.strip()) == 0:
                 # whitespace only lines are not valid syntax in Python's
                 # interactive mode='single', who knew?! Just ignore them.
                 # this might cause actual sytax errors to have bad line
                 # numbers reported, but should only effect interactive mode
                 del lines[idx]
                 last_error_line = last_error_col = -1
                 input = '\n'.join(lines)
                 continue
             maxcol = self._find_next_break(line, last_error_col)
             sbpline = subproc_toks(line,
                                    returnline=True,
                                    maxcol=maxcol,
                                    lexer=self.parser.lexer)
             if sbpline is None:
                 # subprocess line had no valid tokens, likely because
                 # it only contained a comment.
                 del lines[idx]
                 last_error_line = last_error_col = -1
                 input = '\n'.join(lines)
                 continue
             else:
                 lines[idx] = sbpline
             last_error_col += 3
             input = '\n'.join(lines)
     return tree

    def _parse_ctx_free(self, input, mode='exec'):
        last_error_line = last_error_col = -1
        parsed = False
        original_error = None
        while not parsed:
            try:
                tree = self.parser.parse(input,
                                         filename=self.filename,
                                         mode=mode,
                                         debug_level=(self.debug_level > 1))
                parsed = True
            except IndentationError as e:
                if original_error is None:
                    raise e
                else:
                    raise original_error
            except SyntaxError as e:
                if original_error is None:
                    original_error = e
                if (e.loc is None) or (last_error_line == e.loc.lineno and
                                       last_error_col in (e.loc.column + 1,
                                                          e.loc.column)):
                    raise original_error
                last_error_col = e.loc.column
                last_error_line = e.loc.lineno
                idx = last_error_line - 1
                lines = input.splitlines()
                line = lines[idx]
                if input.endswith('\n'):
                    lines.append('')
                if len(line.strip()) == 0:
                    # whitespace only lines are not valid syntax in Python's
                    # interactive mode='single', who knew?! Just ignore them.
                    # this might cause actual sytax errors to have bad line
                    # numbers reported, but should only effect interactive mode
                    del lines[idx]
                    last_error_line = last_error_col = -1
                    input = '\n'.join(lines)
                    continue

                if last_error_line > 1 and lines[idx-1].rstrip()[-1:] == ':':
                    # catch non-indented blocks and raise error.
                    prev_indent = len(lines[idx-1]) - len(lines[idx-1].lstrip())
                    curr_indent = len(lines[idx]) - len(lines[idx].lstrip())
                    if prev_indent == curr_indent:
                        raise original_error
                lexer = self.parser.lexer
                maxcol = find_next_break(line, mincol=last_error_col,
                                         lexer=lexer)
                sbpline = subproc_toks(line, returnline=True,
                                       maxcol=maxcol, lexer=lexer)
                if sbpline is None:
                    # subprocess line had no valid tokens,
                    if len(line.partition('#')[0].strip()) == 0:
                        # likely because it only contained a comment.
                        del lines[idx]
                        last_error_line = last_error_col = -1
                        input = '\n'.join(lines)
                        continue
                    else:
                        # or for some other syntax error
                        raise original_error
                elif sbpline[last_error_col:].startswith('![![') or \
                     sbpline.lstrip().startswith('![!['):
                    # if we have already wrapped this in subproc tokens
                    # and it still doesn't work, adding more won't help
                    # anything
                    raise original_error
                else:
                    if self.debug_level:
                        msg = ('{0}:{1}:{2}{3} - {4}\n'
                               '{0}:{1}:{2}{3} + {5}')
                        mstr = '' if maxcol is None else ':' + str(maxcol)
                        msg = msg.format(self.filename, last_error_line,
                                         last_error_col, mstr, line, sbpline)
                        print(msg, file=sys.stderr)
                    lines[idx] = sbpline
                last_error_col += 3
                input = '\n'.join(lines)
        return tree, input

def test_subproc_toks_indent_ls_no_min_semi_nl():
    s = 'ls'
    exp = INDENT + '$[{0}];\n'.format(s)
    obs = subproc_toks(INDENT + s + ';\n', lexer=LEXER, returnline=True)
    assert_equal(exp, obs)

def test_subproc_toks_indent_ls_nl():
    s = 'ls -l'
    exp = INDENT + '$[{0}]\n'.format(s)
    obs = subproc_toks(INDENT + s + '\n', mincol=len(INDENT), lexer=LEXER,
                       returnline=True)
    assert_equal(exp, obs)