in lib/constructs/data-set-enrollment.ts [30:89]
constructor(scope: Construct, id: string, props: FederatedCrawlerTemplateProps) {
super(scope, id);
const databaseObj = require(props.databaseDescriptionPath);
const crawlerObj = require(props.crawlerDescriptionPath);
// import databaseObj from props.databaseDescriptionPath;
// import tablesObj from props.tablesDescriptionPath;
const databaseName = `${databaseObj['Database']['Name']}`
this.glueDatabase = new glue.CfnDatabase(this, 'GlueDatabase', {
catalogId: Aws.ACCOUNT_ID,
databaseInput: {
locationUri: `${databaseObj['Database']['LocationUri']}`,
name: databaseName,
}
});
this.glueRole = new iam.Role(this, `GlueCrawlerRole`, {
assumedBy: new iam.ServicePrincipal('glue.amazonaws.com')
});
this.glueRole.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSGlueServiceRole'));
this.glueRole.addManagedPolicy(iam.ManagedPolicy.fromAwsManagedPolicyName('CloudWatchAgentServerPolicy'));
var protocolPathTuple = databaseObj['Database']['LocationUri'].split("//");
var pathArray = protocolPathTuple[1].split("/");
let s3DataLakePaths = new Array<glue.CfnCrawler.S3TargetProperty>();
this.glueRole.addToPolicy(new iam.PolicyStatement({
actions: ["s3:*"],
resources: [`arn:aws:s3:::${pathArray[0]}`, `arn:aws:s3:::${pathArray[0]}/${pathArray[1]}/*`]
}));
for (let s3Target of crawlerObj['Crawler']['Targets']['S3Targets']) {
s3DataLakePaths.push({
path: s3Target['Path']
//TODO: add exclusions
});
}
this.glueCrawler = new glue.CfnCrawler(this, `awsroda-crawler`,{
name: `${props.dataSetName}_awsroda_crawler`,
targets: {
s3Targets: s3DataLakePaths
},
role: this.glueRole.roleName,
databaseName: databaseName,
schemaChangePolicy: {
deleteBehavior: "DEPRECATE_IN_DATABASE",
updateBehavior: "UPDATE_IN_DATABASE",
},
tablePrefix: "",
classifiers: []
});
}