async function download()

in src/salesforce_downloader.ts [47:80]

• 32 lines of code
• 2 McCabe index (conditional complexity)

async function download(
	config: Config,
	salesforce: Salesforce,
	folder: Folder & S3Folder,
) {
	console.log('Fetching existing files in S3: ', folder.bucket, folder.prefix);
	const contents = await ls(folder);
	const keys =
		contents?.map((r) => {
			return r.Key?.slice(folder.prefix.length);
		}) || [];
	console.log('Ignoring existing files:', keys);

	console.log('Fetching file list from Saleforce.');
	const documents = await salesforce.getDocuments(folder);
	const filtered = documents.filter((d: { Name: string }) => {
		return !keys.includes(d.Name);
	});

	const uploads = filtered.map(
		async (doc: { Name: string; attributes: { url: string } }) => {
			console.log('Starting download of ', doc.Name);
			const dl = salesforce.getStream(`${doc.attributes.url}/Body`);
			const st = new stream.PassThrough();
			dl.pipe(st);
			console.log('Starting upload to S3 ');
			const streamAsString = await getStream(new stream());
			return upload(streamAsString, doc.Name, folder);
		},
	);
	console.log('Performing upload/downloads.');
	const status = await Promise.all(uploads);
	return { [folder.name]: status.map((s) => s.key) };
}