def _walk()

in azure/datalake/store/core.py [0:0]

• 22 lines of code
• 13 McCabe index (conditional complexity)

    def _walk(self, path, invalidate_cache=True, include_dirs=False):
        """
        Walk a path recursively and returns list of files and dirs(if parameter set)

        Parameters
        ----------
        path: str or AzureDLPath
            Path to query
        invalidate_cache: bool
            Whether to invalidate cache
        include_dirs: bool
            Whether to include dirs in return value

        Returns
        -------
        List of files and (optionally) dirs
        """
        ret = list(self._ls(path, invalidate_cache))
        self._emptyDirs = []
        current_subdirs = [f for f in ret if f['type'] != 'FILE']
        while current_subdirs:
            dirs_below_current_level = []
            for apath in current_subdirs:
                try:
                    sub_elements = self._ls(apath['name'], invalidate_cache)
                except FileNotFoundError:
                    # Folder may have been deleted while walk is going on. Infrequent so we can take the linear hit
                    ret.remove(apath)
                    continue
                if not sub_elements:
                    self._emptyDirs.append(apath)
                else:
                    ret.extend(sub_elements)
                    dirs_below_current_level.extend([f for f in sub_elements if f['type'] != 'FILE'])
            current_subdirs = dirs_below_current_level

        if include_dirs:
            return ret
        else:
            return [f for f in ret if f['type'] == 'FILE']