def run_pyflakes(no_warnings = False):
    """ Returns a dictionary mapping pylearn2 .py filepaths
        to outputs from pyflakes.

        Omits files for which there was no output.

        If no_warnings = True, omits pyflakes outputs that don't
        correspond to actual errors.
    """

    files = list_files(".py")

    rval = {}

    for filepath in files:
        output, rc = run_shell_command('pyflakes ' + filepath)
        if 'pyflakes: not found' in output:
            # The return code alone does not make it possible to detect
            # if pyflakes is present or not. When pyflakes is not present,
            # the return code seems to always be 127, but 127 can also be
            # the result of finding 127 warnings in a file.
            # Therefore, we examine the output instead.
            raise RuntimeError("Couldn't run 'pyflakes " + filepath + "'. "
                    "Error code returned:" + str(rc)\
                    + " Output was: " + output)

        output = _filter(output, no_warnings)

        if output is not None:
            rval[filepath] = output

    return rval

def run_pyflakes(no_warnings = False):
    """ Returns a dictionary mapping pylearn2 .py filepaths
        to outputs from pyflakes.

        Omits files for which there was no output.

        If no_warnings = True, omits pyflakes outputs that don't
        correspond to actual errors.
    """

    files = list_files(".py")

    rval = {}

    for filepath in files:
        output, rc = run_shell_command('pyflakes '+filepath)
        if rc not in [0,1]:
            #pyflakes will return 1 if you give it an invalid file or if
            #the file contains errors, so it's not clear how to detect if
            #pyflakes failed
            #however, if pyflakes just plain isn't installed we should get 127
            raise RuntimeError("Couldn't run 'pyflakes "+filepath+"'."\
                    + "Output was: "+output)

        output = _filter(output, no_warnings)

        if output is not None:
            rval[filepath] = output

    return rval

def test_format():
    format_infractions = []
    for path in list_files(".py"):
        rel_path = os.path.relpath(path, pylearn2.__path__[0])
        if rel_path in whitelist:
            continue
        with open(path) as file:
            for i, line in enumerate(file):
                if len(line.rstrip()) > 79:
                    format_infractions.append((path, i + 1))
    if len(format_infractions) > 0:
        msg = "\n".join('File "%s" line %d has more than 79 characters'
              % (fn, line) for fn, line in format_infractions)
        raise AssertionError("Format not respected:\n%s" % msg)

def test_shebangs():
    # Make sure all scripts that use shebangs use /usr/bin/env
    # (instead of the non-standard /bin/env or hardcoding the path to the
    # interpreter)
    files = list_files('.py')
    for f in files:
        fd = open(f, 'r')
        l = fd.readline()
        fd.close()
        if l.startswith("#!"):
            if not l.startswith("#!/usr/bin/env"):
                print l
                print f
                raise AssertionError("Bad shebang")

def test_format_pep8():
    """
    Test if pep8 is respected.
    """
    pep8_checker = StyleGuide()
    files_to_check = []
    for path in list_files(".py"):
        rel_path = os.path.relpath(path, pylearn2.__path__[0])
        if rel_path in whitelist_pep8:
            continue
        else:
            files_to_check.append(path)
    report = pep8_checker.check_files(files_to_check)
    if report.total_errors > 0:
        raise AssertionError("PEP8 Format not respected")

def test_shebangs():
    # Make sure all scripts that use shebangs use /usr/bin/env
    # (instead of the non-standard /bin/env or hardcoding the path to
    # the interpreter). This test allows any shebang lines that start
    # with /usr/bin/env. Examples:
    #   "#!/usr/bin/env python"
    #   "#! /usr/bin/env python"
    #   "#!/usr/bin/env ipython"
    #   "#!/usr/bin/env ipython --pylab --"
    # etc.
    files = list_files('.py')
    for f in files:
        fd = open(f, 'r')
        l = fd.readline()
        fd.close()
        if l.startswith("#!"):
            if not l[2:].strip().startswith("/usr/bin/env"):
                print(l)
                print(f)
                raise AssertionError("Bad shebang")

def test_format_docstrings():
    """
    Test if docstrings are well formated.
    """
    format_infractions = []

    for path in list_files(".py"):
        rel_path = os.path.relpath(path, pylearn2.__path__[0])
        if rel_path in whitelist_docstrings:
            continue
        try:
            format_infractions.extend(docstring_errors(path))
        except StandardError as e:
            format_infractions.append(["%s failed to run so format cannot "
                                       "be checked. Error message:\n %s" %
                                       (rel_path, e)])

    if len(format_infractions) > 0:
        msg = "\n".join(':'.join(line) for line in format_infractions)
        raise AssertionError("Docstring format not respected:\n%s" % msg)

def verify_format_docstrings():
    """
    Implementation of `test_format_docstrings`. The implementation is
    factored out so it can be placed inside a guard against SkipTest.
    """
    format_infractions = []

    for path in list_files(".py"):
        rel_path = os.path.relpath(path, pylearn2.__path__[0])
        if rel_path in whitelist_docstrings:
            continue
        try:
            format_infractions.extend(docstring_errors(path))
        except StandardError as e:
            format_infractions.append(["%s failed to run so format cannot "
                                       "be checked. Error message:\n %s" %
                                       (rel_path, e)])

    if len(format_infractions) > 0:
        msg = "\n".join(':'.join(line) for line in format_infractions)
        raise AssertionError("Docstring format not respected:\n%s" % msg)

def test_format_pep8():
    """
    Test if pep8 is respected.
    """
    format_infractions = []
    whitespace_infractions = []
    for path in list_files(".py"):
        rel_path = os.path.relpath(path, pylearn2.__path__[0])
        if rel_path in whitelist_pep8:
            continue
        with open(path) as file:
            for i, line in enumerate(file):
                line = line.rstrip("\r\n")
                if len(line) > 79:
                    format_infractions.append((path, i + 1))
                if line.endswith(" ") or line.endswith("\t"):
                    whitespace_infractions.append((path, i + 1))
    if len(format_infractions) + len(whitespace_infractions) > 0:
        msg = "\n".join('File "%s" line %d has more than 79 characters' % (fn, line) for fn, line in format_infractions)
        msg += "\n".join('File "%s" line %d ends with whitespace' % (fn, line) for fn, line in whitespace_infractions)
        raise AssertionError("Format not respected:\n%s" % msg)

def run_pyflakes(no_warnings=False):
    """
    Return a description of all errors pyflakes finds in Pylearn2.

    Parameters
    ----------
    no_warnings : bool
        If True, omits pyflakes outputs that don't correspond to actual
        errors.

    Returns
    -------
    rval : dict
        Keys are pylearn2 .py filepaths
        Values are outputs from pyflakes
    """

    files = list_files(".py")

    rval = {}

    for filepath in files:
        output, rc = run_shell_command('pyflakes ' + filepath)
        output = output.decode(sys.getdefaultencoding())
        if u'pyflakes: not found' in output:
            # The return code alone does not make it possible to detect
            # if pyflakes is present or not. When pyflakes is not present,
            # the return code seems to always be 127, but 127 can also be
            # the result of finding 127 warnings in a file.
            # Therefore, we examine the output instead.
            raise RuntimeError("Couldn't run 'pyflakes " + filepath + "'. "
                               "Error code returned:" + str(rc) +
                               " Output was: " + output)

        output = _filter(output, no_warnings)

        if output is not None:
            rval[filepath] = output

    return rval

def print_files_information_pep8():
    """
    Print the list of files which can be removed from the whitelist and the
    list of files which do not respect PEP8 formatting that aren't in the
    whitelist
    """
    infracting_files = []
    non_infracting_files = []
    pep8_checker = StyleGuide(quiet=True)
    for path in list_files(".py"):
        number_of_infractions = pep8_checker.input_file(path)
        rel_path = os.path.relpath(path, pylearn2.__path__[0])
        if number_of_infractions > 0:
            if rel_path not in whitelist_pep8:
                infracting_files.append(path)
        else:
            if rel_path in whitelist_pep8:
                non_infracting_files.append(path)
    print "Files that must be corrected or added to whitelist:"
    for file in infracting_files:
        print file
    print "Files that can be removed from whitelist:"
    for file in non_infracting_files:
        print file