in backend/lambdas/tasks/generate_queries.py [0:0]
def column_mapper(col):
"""
Function to map Columns from AWS Glue schema to tree
Example 1:
{ Name: "Name", Type: "int" } =>
{ name: "Name", type: "int", canBeIdentifier: true }
Example 2:
{ Name: "complex", Type: "struct<a:string,b:struct<c:int>>"} =>
{ name: "complex", type: "struct", children: [
{ name: "a", type: "string", canBeIdentifier: false},
{ name: "b", type: "struct", children: [
{ name: "c", type: "int", canBeIdentifier: false}
], canBeIdentifier: false}
], canBeIdentifier: false}
"""
prefix = suffix = None
result_type = col["Type"]
has_children = False
if result_type.startswith(ARRAYSTRUCT_PREFIX):
result_type = ARRAYSTRUCT
prefix = ARRAYSTRUCT_PREFIX
suffix = ARRAYSTRUCT_SUFFIX
has_children = True
elif result_type.startswith(STRUCT_PREFIX):
result_type = STRUCT
prefix = STRUCT_PREFIX
suffix = STRUCT_SUFFIX
has_children = True
type_is_decimal_with_precision = result_type.startswith("decimal(")
result = {
"Name": col["Name"],
"Type": result_type,
"CanBeIdentifier": col["CanBeIdentifier"]
if "CanBeIdentifier" in col
else result_type in ALLOWED_TYPES or type_is_decimal_with_precision,
}
if has_children:
result["Children"] = []
children_to_parse = get_inner_children(col["Type"], prefix, suffix)
while len(children_to_parse) > 0:
sep = ":"
name = children_to_parse[0 : children_to_parse.index(sep)]
rest = children_to_parse[len(name) + len(sep) :]
nested_type = "other"
if rest.startswith(STRUCT_PREFIX):
nested_type = STRUCT
elif rest.startswith(ARRAYSTRUCT_PREFIX):
nested_type = ARRAYSTRUCT
c_type = (
get_nested_type(rest)
if nested_type == "other"
else get_nested_children(rest, nested_type)
)
result["Children"].append(
column_mapper(
{
"Name": name,
"Type": c_type,
"CanBeIdentifier": c_type in ALLOWED_TYPES,
}
)
)
children_to_parse = children_to_parse[len(name) + len(sep) + len(c_type) :]
if children_to_parse.startswith(","):
children_to_parse = children_to_parse[1:]
if result_type != STRUCT:
set_no_identifier_to_node_and_its_children(result)
return result