tools/check-tf-plan.py

#!/usr/bin/env python3

# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys
import json
import glob
import shutil
import requests
from python_terraform import Terraform
from urllib.parse import unquote

def main(PR):

    TOKEN             = os.getenv('GITHUB_TOKEN')
    GITHUB_WORKSPACE  = os.getenv('GITHUB_WORKSPACE')
    GITHUB_REPOSITORY = os.getenv('GITHUB_REPOSITORY')


    # Get Added / Modified files in PR
    modified_files, modified_files_raw, removed_files = pr_files(GITHUB_REPOSITORY, PR)

    # Get Working directories to run TF Plan on
    working_directories = get_updated_modules(modified_files, removed_files)

    # Loop through all the identified working directories
    # Deleting added/modified & removed files
    try:
        for dir in working_directories:
            print("----------> RUN FOR: " + dir)


            try:
                # IF MODULE EXISTS: Copying main directory in temp folder
                shutil.copytree(GITHUB_WORKSPACE+'/'+dir, os.getcwd()+'/temp/'+dir)

                # Deleting added/modified & removed files
                for mfile in modified_files:
                    if os.path.exists(os.getcwd()+'/temp/'+mfile):
                        print("Deleting file: " + mfile)
                        os.remove(os.getcwd()+'/temp/'+mfile)

                for rfile in removed_files:
                    if os.path.exists(os.getcwd()+'/temp/'+rfile):
                        print("Deleting file: " + rfile)
                        os.remove(os.getcwd()+'/temp/'+rfile)
            except:
                # IF MODULE DONOT EXISTS: Creating temp module folder
                os.makedirs(os.getcwd()+'/temp/'+dir)

    except requests.exceptions.RequestException as e: 
        print('No working directory with TF configs in PR.')
        raise SystemExit(e)

    # Loop through all the identified working directories
    # Download added/modified files
    try:
      
        for dir in working_directories:
            print("Module: " + dir)

            # Download added/modified files
            for file in modified_files:
        
                if dir in file:
                    print("File: " + file)
                    for raw in modified_files_raw:
                        # print("Raw: " + raw)
                        # print("Raw Decoded: " + unquote(raw))
                              
                        if file in unquote(raw):
                            
                            print("Downloading file: " + unquote(raw))
                            downloadprfiles(unquote(raw), file, os.getcwd()+'/temp/'+os.path.dirname(file))
                            break

    except requests.exceptions.RequestException as e: 
        print('No working directory with TF configs in PR.')
        raise SystemExit(e)


    # Loop through all the identified working directories
    # Run Terraform Plan
    try:
        for dir in working_directories:

            # print('****************************')
            # print(glob.glob(os.getcwd() + '/temp/' + dir+'/*'))
            # print('****************************')
            # print(glob.glob(os.getcwd() + '/temp/' + dir+'/*/*'))

            # Running Terraform Init & Terraform Plan
            comment, status = tf(os.getcwd() + '/temp/' + dir)
            comment = comment + ' for: **' + dir + '** !'

            # Commenting on the PR
            commentpr(GITHUB_REPOSITORY, PR, comment, TOKEN)
            if(status == 'fail'):
                sys.exit('Terraform Init or Terraform Plan FAILED for: '+ dir)

    except requests.exceptions.RequestException as e: 
        print('No working directory with TF configs in PR.')
        raise SystemExit(e)

def pr_files(GITHUB_REPOSITORY,pr):
    removed_files = []
    modified_files = []
    modified_files_raw = []
    try:
        response = requests.get('https://api.github.com/repos/'+ GITHUB_REPOSITORY +'/pulls/'+ str(pr) +'/files')
        for file in response.json():
            if(file['status'] == 'removed'):
              print("Removed File: " + file['filename'])
              removed_files.append(file['filename'])
            else:
              print("Added/Modified File: " + file['filename'])
              modified_files.append(file['filename'])
              modified_files_raw.append(file['raw_url'])

        return modified_files, modified_files_raw, removed_files
    except requests.exceptions.RequestException as e: 
        raise SystemExit(e)  


def downloadprfiles(raw, file, path):

  # print(path)
  if not os.path.exists(path):
      os.makedirs(path)

  # print('Beginning file download with requests')
  r = requests.get(raw)
  with open(path + '/' + os.path.basename(file), 'wb') as f:
      f.write(r.content)

  # Retrieve HTTP meta-data
  # print(r.status_code)
  # print(r.headers['content-type'])
  # print(r.encoding)


def get_updated_modules(modified_files, removed_files):
  modified_files_dir = []
  removed_files_dir = []

  for file in modified_files:
    modified_files_dir.append(os.path.dirname(file))

  for file in removed_files:
    removed_files_dir.append(os.path.dirname(file))

  working_directories = modified_files_dir + removed_files_dir
  working_directories = list(set(working_directories))

#   print("Working Directories:")
#   print(working_directories)
  
  modules = [x for x in working_directories if x.startswith('modules/')]
  modules = [x for x in modules if x.count('/') == 1]
  print("Modules Updated:")
  print(modules)

  return modules


def tf(dir):
  tr = Terraform(working_dir=dir)

  return_code_init, stdout_init, stderr_init = tr.init_cmd(capture_output=False)
  
  if "secure_data_warehouse" in dir:
    # Special case in SDW module as it require additional mandatory variables compared to any other modules.
    return_code_plan, stdout_plan, stderr_plan = tr.plan_cmd(capture_output=False,var={'billing_account_id':'ABCD-EFGH-IJKL-MNOP', 'organization_id':'1234567890', 'random_id': '1234', 'data_analyst_group': 'data_analyst_group@example.com', 'data_engineer_group': 'data_engineer_group@example.com', 'security_administrator_group': 'security_administrator_group@example.com', 'network_administrator_group': 'network_administrator_group@example.com', 'security_analyst_group': 'security_analyst_group@example.com', 'perimeter_additional_members': ['demouser@example.com','demosa@service.gserviceaccount.com'], 'secure_datawarehouse_service_acccount': 'radlab-module-creator-sa@project-id.iam.gserviceaccount.com'})
  else:
    return_code_plan, stdout_plan, stderr_plan = tr.plan_cmd(capture_output=False,var={'billing_account_id':'ABCD-EFGH-IJKL-MNOP', 'organization_id':'1234567890', 'random_id': '1234'})
  
  path = os.getcwd()+'/temp/'
  if(return_code_init == 1):
    comment = 'Terraform Init FAILED'
    status = 'fail'
  if(return_code_plan == 1):
    comment = 'Terraform Plan FAILED'
    status = 'fail'
  else: 
    comment = 'Terraform Init & Terraform Plan SUCCESSFUL'
    status = 'pass'
  
  return comment, status


def commentpr(GITHUB_REPOSITORY, pr, comment, TOKEN):
    headers = {'Authorization': f'token {TOKEN}', 'Accept': 'application/vnd.github.v3+json'}
    # print(comment)
    data = {"body":comment}
    try:
        response  = requests.post('https://api.github.com/repos/'+ GITHUB_REPOSITORY +'/issues/'+ str(pr) +'/comments', data=json.dumps(data), headers=headers)
        # print(response.text)
    except requests.exceptions.RequestException as e: 
        raise SystemExit(e)

if __name__ == '__main__':

  if len(sys.argv) != 2:
    raise SystemExit('No PR passed.')
  main(sys.argv[1])