in Xcode/AppleCookieDownloader.py [0:0]
def download(self, url, curl_opts, output, request_headers, allow_failure=False):
"""Run a download with curl."""
# construct curl command.
curl_cmd = [
self.env["CURL_PATH"],
"--silent",
"--show-error",
"--no-buffer",
"--fail",
"--dump-header",
"-",
"--speed-time",
"30",
"--location",
"--url",
url,
"--output",
output,
]
if request_headers:
for header, value in request_headers.items():
curl_cmd.extend(["--header", "%s: %s" % (header, value)])
if curl_opts:
for item in curl_opts:
curl_cmd.extend([item])
# Open URL.
proc = subprocess.Popen(
curl_cmd,
shell=False,
bufsize=1,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
donewithheaders = False
maxheaders = 15
header = {}
header["http_result_code"] = "000"
header["http_result_description"] = ""
while True:
if not donewithheaders:
info = proc.stdout.readline().decode().strip("\r\n")
if info.startswith("HTTP/"):
try:
header["http_result_code"] = info.split(None, 2)[1]
header["http_result_description"] = info.split(None, 2)[2]
except IndexError:
pass
elif ": " in info:
# got a header line
part = info.split(None, 1)
fieldname = part[0].rstrip(":").lower()
try:
header[fieldname] = part[1]
except IndexError:
header[fieldname] = ""
elif info == "":
# we got an empty line; end of headers (or curl exited)
if header.get("http_result_code") in [
"301",
"302",
"303",
"307",
"308",
]:
# redirect, so more headers are coming.
# Throw away the headers we've received so far
header = {}
header["http_result_code"] = "000"
header["http_result_description"] = ""
else:
donewithheaders = True
else:
time.sleep(0.1)
if proc.poll() is not None:
# For small download files curl may exit before all headers
# have been parsed, don't immediately exit.
maxheaders -= 1
if donewithheaders or maxheaders <= 0:
break
retcode = proc.poll()
if (
retcode and not allow_failure
): # Non-zero exit code from curl => problem with download
curlerr = ""
try:
curlerr = proc.stderr.read().rstrip("\n")
curlerr = curlerr.split(None, 2)[2]
except IndexError:
pass
raise ProcessorError("Curl failure: %s (exit code %s)" % (curlerr, retcode))