public Pair loadTableMetadata()

in src/spark-project/engine-spark/src/main/java/org/apache/kylin/engine/spark/source/NSparkMetadataExplorer.java [243:326]
75 lines of code
8 McCabe index (conditional complexity)

    public Pair<TableDesc, TableExtDesc> loadTableMetadata(final String database, String tableName, String prj)
            throws Exception {
        KylinConfig config = KylinConfig.getInstanceFromEnv();
        NTableMetadataManager metaMgr = NTableMetadataManager.getInstance(config, prj);

        NSparkTableMeta tableMeta = getTableMetaExplorer().getSparkTableMeta(database, tableName);
        TableDesc tableDesc = metaMgr.getTableDesc(database + "." + tableName);

        // make a new TableDesc instance, don't modify the one in use
        if (tableDesc == null) {
            tableDesc = new TableDesc();
            tableDesc.setDatabase(database.toUpperCase(Locale.ROOT));
            tableDesc.setName(tableName.toUpperCase(Locale.ROOT));
            tableDesc.setUuid(RandomUtil.randomUUIDStr());
            tableDesc.setLastModified(0);
        } else {
            tableDesc = new TableDesc(tableDesc);
        }

        if (tableMeta.tableType != null) {
            tableDesc.setTableType(tableMeta.tableType);
        }
        //set table type = spark
        tableDesc.setSourceType(tableMeta.sourceType);
        tableDesc.setTransactional(tableMeta.isTransactional);
        tableDesc.setRangePartition(tableMeta.isRangePartition);
        tableDesc.setTableComment(tableMeta.tableComment);

        Set<String> partColumnSet = Optional.ofNullable(tableMeta.partitionColumns) //
                .orElseGet(Collections::emptyList).stream().map(field -> field.name) //
                .collect(Collectors.toSet());
        int columnNumber = tableMeta.allColumns.size();
        List<ColumnDesc> columns = new ArrayList<>(columnNumber);
        for (int i = 0; i < columnNumber; i++) {
            NSparkTableMeta.SparkTableColumnMeta field = tableMeta.allColumns.get(i);
            ColumnDesc cdesc = new ColumnDesc();
            cdesc.setName(field.name.toUpperCase(Locale.ROOT));
            cdesc.setCaseSensitiveName(field.name);
            // use "double" in kylin for "float"
            if ("float".equalsIgnoreCase(field.dataType)) {
                cdesc.setDatatype("double");
            } else {
                cdesc.setDatatype(field.dataType);
            }
            cdesc.setId(String.valueOf(i + 1));
            cdesc.setComment(field.comment);
            cdesc.setPartitioned(partColumnSet.contains(field.name));
            columns.add(cdesc);
        }
        tableDesc.setColumns(columns.toArray(new ColumnDesc[columnNumber]));
        List<String> partCols = tableMeta.partitionColumns.stream().map(col -> col.name).collect(Collectors.toList());
        if (!partCols.isEmpty()) {
            tableDesc.setPartitionColumn(partCols.get(0).toUpperCase(Locale.ROOT));
        } else {
            tableDesc.setPartitionColumn(null);
        }
        StringBuilder partitionColumnBuilder = new StringBuilder();
        for (int i = 0, n = tableMeta.partitionColumns.size(); i < n; i++) {
            if (i > 0)
                partitionColumnBuilder.append(", ");
            partitionColumnBuilder.append(tableMeta.partitionColumns.get(i).name.toUpperCase(Locale.ROOT));
        }

        TableExtDesc tableExtDesc = new TableExtDesc();
        tableExtDesc.setIdentity(tableDesc.getIdentity());
        tableExtDesc.setUuid(RandomUtil.randomUUIDStr());
        tableExtDesc.setLastModified(0);
        tableExtDesc.init(prj);

        tableExtDesc.addDataSourceProp(TableExtDesc.LOCATION_PROPERTY_KEY, tableMeta.sdLocation);
        tableExtDesc.addDataSourceProp("owner", tableMeta.owner);
        tableExtDesc.addDataSourceProp("create_time", tableMeta.createTime);
        tableExtDesc.addDataSourceProp("last_access_time", tableMeta.lastAccessTime);
        tableExtDesc.addDataSourceProp("partition_column", partitionColumnBuilder.toString());
        tableExtDesc.addDataSourceProp("total_file_size", String.valueOf(tableMeta.fileSize));
        tableExtDesc.addDataSourceProp("total_file_number", String.valueOf(tableMeta.fileNum));
        tableExtDesc.addDataSourceProp("hive_inputFormat", tableMeta.sdInputFormat);
        tableExtDesc.addDataSourceProp("hive_outputFormat", tableMeta.sdOutputFormat);
        ObsConfig obsConfig = ObsConfig.getByLocation(tableMeta.sdLocation).orElse(ObsConfig.S3);
        tableExtDesc.addDataSourceProp(obsConfig.getRolePropertiesKey(), tableMeta.roleArn);
        tableExtDesc.addDataSourceProp(obsConfig.getEndpointPropertiesKey(), tableMeta.endpoint);
        tableExtDesc.addDataSourceProp(obsConfig.getRegionPropertiesKey(), tableMeta.region);
        return Pair.newPair(tableDesc, tableExtDesc);
    }