def iter_files(root, exts=None, recursive=False):
    """
    Iterate over file paths within root filtered by specified extensions.

    :param str root: Root folder to start collecting files
    :param iterable exts: Restrict results to given file extensions
    :param bool recursive: Wether to walk the complete directory tree
    :rtype collections.Iterable[str]: absolute file paths with given extensions
    """

    if exts is not None:
        exts = set((x.lower() for x in exts))

    def matches(e):
        return (exts is None) or (e in exts)

    if recursive is False:
        for entry in scandir(root):
            if has_scandir:
                ext = splitext(entry.name)[-1].lstrip('.').lower()
                if entry.is_file() and matches(ext):
                    yield entry.path
            else:
                ext = splitext(entry)[-1].lstrip('.').lower()
                if not isdir(entry) and matches(ext):
                    yield join(root, entry)
    else:
        for root, folders, files in walk(root):
            for f in files:
                ext = splitext(f)[-1].lstrip('.').lower()
                if matches(ext):
                    yield join(root, f)

    def download_gallery(self,
                         gallery_url,
                         download_path,
                         options,
                         status,
                         root=False):
        """
            Download an complete gallery, calls download_gallery_images
            for the actual image download.

            This creates the folder structure, and walks through it
            calling download_gallery_images to download the images.
        """
        current_webpage = common.fetch_webpage(session=self.session,
                                               url=gallery_url,
                                               timeout=45)
        soup = BeautifulSoup(current_webpage)
        #
        #   Grab the main web page from the URL to fetch
        #
        #   Search for folders
        folder_list = self.search_for_folders(soup_bowl=soup)
        for (subgallery_name, subgallery_url) in folder_list:
             #
             #   Process the folder list, and download
             #   the images for the subfolders
             #
            if options.downloadlimit > 0 and \
                    status.return_downloads() >= options.downloadlimit:
                print "X",
                return status
            if subgallery_name != None:
                subgallery_dl_path = download_path + os.sep +\
                    common.clean_filename(subgallery_name) + os.sep
            if subgallery_url != gallery_url:
                #
                #   Clubs typically have the featured gallery which points to
                #   itself and can cause a recursion loop
                #
                status = self.download_gallery(subgallery_url,
                                               subgallery_dl_path,
                                               options,
                                               status,
                                               root=False)
            time.sleep(1)
        gallery_name = soup.title.text
        gallery_name = gallery_name[0:gallery_name.find(" by ")].strip()

        if root:
            for root, dirnames, filenames in scandir.walk(download_path):
                for filename in filenames:
                    self.root_checker[filename.lower().strip()] = True

        status = self.download_gallery_images(gallery_url,
                                              download_path,
                                              options,
                                              status,
                                              root=root)

        return status

    def process_directory(self, path, recursive=True, timing=True):
        """ Processes the specified directory, extracting file sizes for each file and
            adding to a file extension indexed dictionary.
        :param path: the path to analyse
        :param recursive: true if processing should include sub-directories
        :param timing: true if path should be preprocessed to provide guidance on run-time
        :return:
        """

        # get number of files - have to scan dir once to start with
        print "\n\rProcessing {0}...".format(path)
        bar = progressbar.ProgressBar(max_value=progressbar.UnknownLength)

        # If user wants more accurate timing, preprocess directory to count files
        if timing:
            numfiles = self._count_dirs(path, recursive)
            bar.start(numfiles)

        # grab file extension and file sizes across all files in the specified directory
        for root, dirs, files in scandir.walk(path, followlinks=False):
            # if only processing the top level, remove dirs so os.walk doesn't progress further
            if not recursive:
                del dirs[:]

            for name in files:
                filename = os.path.join(root, name)
                fname, fext = os.path.splitext(filename)
                fext = self._convert_extension(fext.lower())   # lowercase all filenames

                if os.path.exists(filename):
                    if fext not in self.filestats:
                        self.filestats[fext] = RunningStat()
                    self.filestats[fext].add(os.stat(filename).st_size)
                bar.update(bar.value+1)
        bar.finish()

def mywalk(top, skipdirs=['.snapshot',]):
    """ returns subset of os.walk  """
    for root, dirs, files in walk(top,topdown=True,onerror=walkerr): 
        for skipdir in skipdirs:
            if skipdir in dirs:
                dirs.remove(skipdir)  # don't visit this directory 
        yield root, dirs, files 

def find_files(location, pattern, ignore_dirs=[], maxdepth=float('inf')):
    """ Find paths to images on disk matching an given pattern

    Args:
        location (str): root directory to search
        pattern (str): glob style pattern to search for
        ignore_dirs (iterable): list of directories to ignore from search
        maxdepth (int): maximum depth to recursively search

    Returns:
        list: list of files within location matching pattern

    """
    results = []

    if isinstance(ignore_dirs, str):
        ignore_dirs = list(ignore_dirs)

    location = os.path.normpath(location)
    num_sep = location.count(os.path.sep) - 1

    for root, dirs, files in walk(location, followlinks=True):
        if ignore_dirs:
            dirs[:] = [d for d in dirs if d not in ignore_dirs]

        depth = root.count(os.path.sep) - num_sep
        if depth > maxdepth:
            dirs[:] = []
            files[:] = []

        for fname in fnmatch.filter(files, pattern):
            results.append(os.path.abspath(os.path.join(root, fname)))

    return results

def find_file(image):
    matches = []
    for root, dirnames, filenames in scandir.walk(assetDir):
        for filename in fnmatch.filter(filenames, image):
            matches.append(os.path.join(root, filename))

    return matches

def find_mp3s(path):
    """
    - path: directory path containing mp3s, or a text file playlist
    """
    path = os.path.abspath(os.path.expanduser(path))
    if os.path.isfile(path):
        with open(path, 'r') as fp:
            text = fp.read()

        results = [mp3 for mp3 in re.split('\r?\n', text) if mp3]
    elif os.path.isdir(path):
        results = []

        for dirpath, dirnames, filenames in walk(path):
            files = [f for f in filenames if f.lower().endswith('.mp3')]

            if not files:
                continue

            results.extend([os.path.join(dirpath, f) for f in files])
    else:
        print('{} is not a file or a directory'.format(repr(path)))
        import ipdb; ipdb.set_trace()

    return results

def locale_directory_path(checkout_path, locale_code, parent_directories=None):
    """
    Path to the directory where strings for the given locale are
    stored.
    """
    possible_paths = []

    # Check paths that use underscore as locale/country code separator
    locale_code_variants = [locale_code, locale_code.replace('-', '_')]

    # Optimization for directories with a lot of paths: if parent_directories
    # is provided, we simply join it with locale_code and check if path exists
    for parent_directory in parent_directories:
        for locale in locale_code_variants:
            candidate = os.path.join(parent_directory, locale)
            if os.path.exists(candidate):
                possible_paths.append(candidate)

    if not possible_paths:
        for root, dirnames, filenames in scandir.walk(checkout_path):
            for locale in locale_code_variants:
                if locale in dirnames:
                    possible_paths.append(os.path.join(root, locale))

    for possible_path in possible_paths:
        if directory_contains_resources(possible_path):
            return possible_path

    # If locale directory empty (asymmetric formats)
    if possible_paths:
        return possible_paths[0]

    raise IOError('Directory for locale `{0}` not found'.format(
                  locale_code or 'source'))

    def source_directory_path(self):
        """
        Path to the directory where source strings are stored.

        Paths are identified using a scoring system; more likely
        directory names get higher scores, as do directories with
        formats that only used for source strings.
        """
        # If source repository explicitly marked
        source_repository = self.db_project.source_repository
        if source_repository.source_repo:
            return source_repository.checkout_path

        possible_sources = []
        for root, dirnames, filenames in scandir.walk(self.checkout_path):
            for dirname in dirnames:
                if dirname in self.SOURCE_DIR_NAMES:
                    score = self.SOURCE_DIR_SCORES[dirname]

                    # Ensure the matched directory contains resources.
                    directory_path = os.path.join(root, dirname)
                    if directory_contains_resources(directory_path):
                        # Extra points for source resources!
                        if directory_contains_resources(directory_path, source_only=True):
                            score += 3

                        possible_sources.append((directory_path, score))

        if possible_sources:
            return max(possible_sources, key=lambda s: s[1])[0]
        else:
            raise MissingSourceDirectoryError(
                'No source directory found for project {0}'.format(self.db_project.slug)
            )

 def extractCBR(self, targetdir):
     cbrFile = rarfile.RarFile(self.origFileName)
     cbrFile.extractall(targetdir)
     for root, dirnames, filenames in walk(targetdir):
         for filename in filenames:
             if filename.startswith('__MACOSX') or filename.endswith('.DS_Store') or filename.endswith('humbs.db'):
                 os.remove(os.path.join(root, filename))

def recursive_gallery_check(path):
	"""
	Recursively checks a folder for any potential galleries
	Returns a list of paths for directories and a list of tuples where first
	index is path to gallery in archive and second index is path to archive.
	Like this:
	["C:path/to/g"] and [("path/to/g/in/a", "C:path/to/a")]
	"""
	gallery_dirs = []
	gallery_arch = []
	for root, subfolders, files in scandir.walk(path):
		if files:
			for f in files:
				if f.endswith(ARCHIVE_FILES):
					arch_path = os.path.join(root, f)
					for g in check_archive(arch_path):
						gallery_arch.append((g, arch_path))
									
			if not subfolders:
				if not files:
					continue
				gallery_probability = len(files)
				for f in files:
					if not f.lower().endswith(IMG_FILES):
						gallery_probability -= 1
				if gallery_probability >= (len(files)*0.8):
					gallery_dirs.append(root)
	return gallery_dirs, gallery_arch

def extract_xpi(xpi, path, expand=False, verify=True):
    """
    If expand is given, will look inside the expanded file
    and find anything in the allow list and try and expand it as well.
    It will do up to 10 iterations, after that you are on your own.

    It will replace the expanded file with a directory and the expanded
    contents. If you have 'foo.jar', that contains 'some-image.jpg', then
    it will create a folder, foo.jar, with an image inside.
    """
    expand_allow_list = ['.crx', '.jar', '.xpi', '.zip']
    tempdir = extract_zip(xpi)
    all_files = get_all_files(tempdir)

    if expand:
        for x in xrange(0, 10):
            flag = False
            for root, dirs, files in scandir.walk(tempdir):
                for name in files:
                    if os.path.splitext(name)[1] in expand_allow_list:
                        src = os.path.join(root, name)
                        if not os.path.isdir(src):
                            dest = extract_zip(
                                src, remove=True, raise_on_failure=False)
                            all_files.extend(get_all_files(
                                dest, strip_prefix=tempdir, prefix=src))
                            if dest:
                                copy_over(dest, src)
                                flag = True
            if not flag:
                break

    copy_over(tempdir, path)
    return all_files

def get_audio_files(location):
    for path, dirs, files in walk(location):
        for f in files:
            if (f.endswith('.m4a') or f.endswith('.mp3')
                    or f.endswith('.flac')) and not f.startswith('.'):
                print 'Got audio file:', f
                yield os.path.join(path, f)

    def _find_changes(self):
        """
        Walks the filesystem. Identifies noteworthy files -- those
        that were added, removed, or changed (size, mtime or type).

        Returns a 3-tuple of sets of HashEntry objects:
        [0] added files
        [1] removed files
        [2] modified files

        self.entries is not modified; this method only reports changes.
        """
        added = set()
        modified = set()
        existing_files = set()
        for dirpath_str, _, filenames in walk(str(self.path)):
            dirpath = Path(dirpath_str)
            for filename in filenames:
                if filename == DB_FILENAME:
                    continue
                abs_filename = (dirpath / filename).absolute()
                if abs_filename in self.entries:
                    entry = self.entries[abs_filename]
                    existing_files.add(entry)
                    st = lstat(str(abs_filename))
                    if entry != st:
                        modified.add(entry)
                else:
                    entry = HashEntry(abs_filename)
                    entry.update_attrs()
                    added.add(entry)
        removed = set(self.entries.values()) - existing_files
        return added, removed, modified

def list_dirs(d, suffix=None, reverse=False):
    """A generator that works much like :py:func:`os.listdir`, only
    recursively (and only returns files, not directories).

    :param d: The directory to start in
    :type d: str
    :param suffix: Only return files with the given suffix
    :type suffix: str or list
    :param reverse: Returns result sorted in reverse alphabetic order
    :param type:
    :returns: the full path (starting from d) of each matching file
    :rtype: generator

    """
    try:
        from scandir import walk
    except ImportError:
        from os import walk
    if isinstance(suffix, str):
        suffix = [suffix]
    for (dirpath, dirnames, filenames) in walk(d, topdown=True):
        dirnames.sort(reverse=reverse, key=split_numalpha)
        for filename in sorted(filenames, key=split_numalpha, reverse=reverse):
            fullpath = dirpath + os.sep + filename
            if suffix:
                for s in suffix:
                    if fullpath.endswith(s):
                        yield fullpath
            else:
                yield fullpath

    def parse_dir(dir): 
        ignores = Parser.load_ignores(dir)
        ignores.extend([".svn", ".hg", ".git"])

        def callback(res):
            dependencies.extend(res)

        def is_ignored(res, is_dir=False):
            if is_dir:
                res = res + "/"
            for i in ignores:
                if fnmatch.fnmatch(res, i) or res.startswith(i):
                    return True
            return False

        def find_ignored(reslist, is_dir=False):
            return [res for res in reslist if is_ignored(res, is_dir)]

        pool = ThreadPool(processes=Parser.concurrency)
        dependencies = []

        for root, dirs, files in scandir.walk(dir):
            for d in find_ignored(dirs, True):
                logging.debug("%s is blacklisted" % d)
                dirs.remove(d)
            for f in find_ignored(files):
                logging.debug("%s is blacklisted" % d)
                files.remove(f)
            for name in files:
                pool.apply_async(Parser.parse_file, args = (os.path.join(root, name),), callback = callback)

        pool.close()
        pool.join()
        return dependencies

 def scan_folder(self, path):
     (pathS, directoriesS, filesS) = ((), (), ())
     try:
         for (pathS, directoriesS, filesS) in myScandir.walk(path):
             break
         return (pathS, directoriesS, filesS)
     except os.error:
         log("Path", path, "is not accessible.")

def get_file_names():
    filenames = []

    for folder, _, files in walk(icloudpath):
        for filename in files:
            filenames.append((folder + '/' + filename)[len(icloudpath) + 1:])

    return filenames

def scan_files(path):
    """
    Recursively scan a directory to find all files with the given extension.
    """
    ext = tuple(MEDIA_EXT)
    for root, _, files in scandir.walk(path):
        for f in files:
            if f.endswith(ext):
                yield Movie(path=root + '/' + f)

    def list(self):
        self.log('info', "List of available RAT modules:")

        for folder, folders, files in walk(os.path.join(VIPER_ROOT, 'modules/rats/')):
            for file_name in files:
                if not file_name.endswith('.py') or file_name.startswith('__init__'):
                    continue

                self.log('item', os.path.join(folder, file_name))