in datasets/usda_nass_agriculture/pipelines/_images/run_script_kub/script.py [0:0]
def main(directory, host, gcs_bucket, gcs_path):
ftp = FTP(host)
ftp.login()
ftp.cwd(directory)
gzfiles = ftp.nlst()
for file in gzfiles:
if (
file.endswith("gz")
and "zipcode" not in file
and "crop" not in file
and "census2017" not in file
and "environment" not in file
):
logging.info(f"Downloading file {file} ---->")
with open(file, "wb") as f:
ftp.retrbinary(f"RETR {file}", f.write)
upload_gcs(file, gcs_bucket, gcs_path)
os.remove(file)