def concat()

in azure/datalake/store/core.py [0:0]

• 12 lines of code
• 3 McCabe index (conditional complexity)

    def concat(self, outfile, filelist, delete_source=False):
        """ Concatenate a list of files into one new file

        Parameters
        ----------

        outfile: path
            The file which will be concatenated to. If it already exists,
            the extra pieces will be appended.
        filelist: list of paths
            Existing adl files to concatenate, in order
        delete_source: bool (False)
            If True, assume that the paths to concatenate exist alone in a
            directory, and delete that whole directory when done.

        Returns
        -------
        None
        """
        outfile = AzureDLPath(outfile).trim()
        delete = 'true' if delete_source else 'false'
        sourceList = [AzureDLPath(f).as_posix() for f in filelist]
        sources = {}
        sources["sources"] = sourceList

        self.azure.call('MSCONCAT', outfile.as_posix(),
                        data=bytearray(json.dumps(sources, separators=(',', ':')), encoding="utf-8"),
                        deleteSourceDirectory=delete,
                        headers={'Content-Type': "application/json"},
                        retry_policy=NoRetryPolicy())
        self.invalidate_cache(outfile)