datasets/cms_medicare/pipelines/inpatient_charges/pipeline.yaml (441 lines of code) (raw):
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
resources:
- type: bigquery_table
table_id: inpatient_charges_2011
description: "CMS Medicare Inpatient Charges 2011"
- type: bigquery_table
table_id: inpatient_charges_2012
description: "CMS Medicare Inpatient Charges 2012"
- type: bigquery_table
table_id: inpatient_charges_2013
description: "CMS Medicare Inpatient Charges 2013"
- type: bigquery_table
table_id: inpatient_charges_2014
description: "CMS Medicare Inpatient Charges 2014"
- type: bigquery_table
table_id: inpatient_charges_2015
description: "CMS Medicare Inpatient Charges 2015"
dag:
airflow_version: 2
initialize:
dag_id: inpatient_charges
default_args:
owner: "Google"
depends_on_past: False
start_date: '2021-03-01'
max_active_runs: 1
schedule_interval: "@once"
catchup: False
default_view: graph
tasks:
- operator: "KubernetesPodOperator"
description: "Run CSV transform within kubernetes pod"
args:
task_id: "inpatient_2011_transform_csv"
startup_timeout_seconds: 600
name: "cms_medicare_inpatient_charges_2011"
namespace: "composer-user-workloads"
service_account_name: "default"
config_file: "/home/airflow/composer_kube_config"
image_pull_policy: "Always"
image: "{{ var.json.cms_medicare.container_registry.run_csv_transform_kub }}"
env_vars:
SOURCE_URL: "https://www.cms.gov/Research-Statistics-Data-and-Systems/Statistics-Trends-and-Reports/Medicare-Provider-Charge-Data/Downloads/Inpatient_Data_2011_CSV.zip"
SOURCE_FILE: "files/data.zip"
TARGET_FILE: "files/data_output.csv"
TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}"
TARGET_GCS_PATH: "data/cms_medicare/inpatient_charges_2011/data_output.csv"
CSV_HEADERS: >-
["provider_id","provider_name","provider_street_address","provider_city","provider_state","provider_zipcode","drg_definition","hospital_referral_region_description","total_discharges","average_covered_charges","average_total_payments","average_medicare_payments"]
RENAME_MAPPINGS: >-
{"Provider Id": "provider_id","Provider Name": "provider_name","Provider Street Address": "provider_street_address","Provider City": "provider_city","Provider State": "provider_state","Provider Zip Code": "provider_zipcode","DRG Definition": "drg_definition","Hospital Referral Region (HRR) Description": "hospital_referral_region_description","Total Discharges": "total_discharges","Average Covered Charges": "average_covered_charges","Average Total Payments": "average_total_payments","Average Medicare Payments": "average_medicare_payments"}
PIPELINE_NAME: "inpatient_charges_2011"
- operator: "KubernetesPodOperator"
description: "Run CSV transform within kubernetes pod"
args:
task_id: "inpatient_2012_transform_csv"
startup_timeout_seconds: 600
name: "cms_medicare_inpatient_charges_2012"
namespace: "composer-user-workloads"
service_account_name: "default"
config_file: "/home/airflow/composer_kube_config"
image_pull_policy: "Always"
image: "{{ var.json.cms_medicare.container_registry.run_csv_transform_kub }}"
env_vars:
SOURCE_URL: "https://www.cms.gov/Research-Statistics-Data-and-Systems/Statistics-Trends-and-Reports/Medicare-Provider-Charge-Data/Downloads/Inpatient_Data_2012_CSV.zip"
SOURCE_FILE: "files/data.zip"
TARGET_FILE: "files/data_output.csv"
TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}"
TARGET_GCS_PATH: "data/cms_medicare/inpatient_charges_2012/data_output.csv"
CSV_HEADERS: >-
["provider_id","provider_name","provider_street_address","provider_city","provider_state","provider_zipcode","drg_definition","hospital_referral_region_description","total_discharges","average_covered_charges","average_total_payments","average_medicare_payments"]
RENAME_MAPPINGS: >-
{"Provider Id": "provider_id","Provider Name": "provider_name","Provider Street Address": "provider_street_address","Provider City": "provider_city","Provider State": "provider_state","Provider Zip Code": "provider_zipcode","DRG Definition": "drg_definition","Hospital Referral Region (HRR) Description": "hospital_referral_region_description","Total Discharges": "total_discharges","Average Covered Charges": "average_covered_charges","Average Total Payments": "average_total_payments","Average Medicare Payments": "average_medicare_payments"}
PIPELINE_NAME: "inpatient_charges_2012"
- operator: "KubernetesPodOperator"
description: "Run CSV transform within kubernetes pod"
args:
task_id: "inpatient_2013_transform_csv"
startup_timeout_seconds: 600
name: "cms_medicare_inpatient_charges_2013"
namespace: "composer-user-workloads"
service_account_name: "default"
config_file: "/home/airflow/composer_kube_config"
image_pull_policy: "Always"
image: "{{ var.json.cms_medicare.container_registry.run_csv_transform_kub }}"
env_vars:
SOURCE_URL: "https://www.cms.gov/Research-Statistics-Data-and-Systems/Statistics-Trends-and-Reports/Medicare-Provider-Charge-Data/Downloads/Inpatient_Data_2013_CSV.zip"
SOURCE_FILE: "files/data.zip"
TARGET_FILE: "files/data_output.csv"
TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}"
TARGET_GCS_PATH: "data/cms_medicare/inpatient_charges_2013/data_output.csv"
CSV_HEADERS: >-
["provider_id","provider_name","provider_street_address","provider_city","provider_state","provider_zipcode","drg_definition","hospital_referral_region_description","total_discharges","average_covered_charges","average_total_payments","average_medicare_payments"]
RENAME_MAPPINGS: >-
{"Provider Id": "provider_id","Provider Name": "provider_name","Provider Street Address": "provider_street_address","Provider City": "provider_city","Provider State": "provider_state","Provider Zip Code": "provider_zipcode","DRG Definition": "drg_definition","Hospital Referral Region (HRR) Description": "hospital_referral_region_description","Total Discharges": "total_discharges","Average Covered Charges": "average_covered_charges","Average Total Payments": "average_total_payments","Average Medicare Payments": "average_medicare_payments"}
PIPELINE_NAME: "inpatient_charges_2013"
- operator: "KubernetesPodOperator"
description: "Run CSV transform within kubernetes pod"
args:
task_id: "inpatient_2014_transform_csv"
startup_timeout_seconds: 600
name: "cms_medicare_inpatient_charges_2014"
namespace: "composer-user-workloads"
service_account_name: "default"
config_file: "/home/airflow/composer_kube_config"
image_pull_policy: "Always"
image: "{{ var.json.cms_medicare.container_registry.run_csv_transform_kub }}"
env_vars:
SOURCE_URL: "https://www.cms.gov/Research-Statistics-Data-and-Systems/Statistics-Trends-and-Reports/Medicare-Provider-Charge-Data/Downloads/Inpatient_Data_2014_CSV.zip"
SOURCE_FILE: "files/data.zip"
TARGET_FILE: "files/data_output.csv"
TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}"
TARGET_GCS_PATH: "data/cms_medicare/inpatient_charges_2014/data_output.csv"
CSV_HEADERS: >-
["provider_id","provider_name","provider_street_address","provider_city","provider_state","provider_zipcode","drg_definition","hospital_referral_region_description","total_discharges","average_covered_charges","average_total_payments","average_medicare_payments"]
RENAME_MAPPINGS: >-
{"Provider Id": "provider_id","Provider Name": "provider_name","Provider Street Address": "provider_street_address","Provider City": "provider_city","Provider State": "provider_state","Provider Zip Code": "provider_zipcode","DRG Definition": "drg_definition","Hospital Referral Region (HRR) Description": "hospital_referral_region_description","Total Discharges": "total_discharges","Average Covered Charges": "average_covered_charges","Average Total Payments": "average_total_payments","Average Medicare Payments": "average_medicare_payments"}
PIPELINE_NAME: "inpatient_charges_2014"
- operator: "KubernetesPodOperator"
description: "Run CSV transform within kubernetes pod"
args:
task_id: "inpatient_2015_transform_csv"
startup_timeout_seconds: 600
name: "cms_medicare_inpatient_charges_2015"
namespace: "composer-user-workloads"
service_account_name: "default"
config_file: "/home/airflow/composer_kube_config"
image_pull_policy: "Always"
image: "{{ var.json.cms_medicare.container_registry.run_csv_transform_kub }}"
env_vars:
SOURCE_URL: "https://www.cms.gov/Research-Statistics-Data-and-Systems/Statistics-Trends-and-Reports/Medicare-Provider-Charge-Data/Downloads/Inpatient_Data_2015_CSV.zip"
SOURCE_FILE: "files/data.zip"
TARGET_FILE: "files/data_output.csv"
TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}"
TARGET_GCS_PATH: "data/cms_medicare/inpatient_charges_2015/data_output.csv"
CSV_HEADERS: >-
["provider_id","provider_name","provider_street_address","provider_city","provider_state","provider_zipcode","drg_definition","hospital_referral_region_description","total_discharges","average_covered_charges","average_total_payments","average_medicare_payments"]
RENAME_MAPPINGS: >-
{"Provider Id": "provider_id","Provider Name": "provider_name","Provider Street Address": "provider_street_address","Provider City": "provider_city","Provider State": "provider_state","Provider Zip Code": "provider_zipcode","DRG Definition": "drg_definition","Hospital Referral Region (HRR) Description": "hospital_referral_region_description","Total Discharges": "total_discharges","Average Covered Charges": "average_covered_charges","Average Total Payments": "average_total_payments","Average Medicare Payments": "average_medicare_payments"}
PIPELINE_NAME: "inpatient_charges_2015"
- operator: "GoogleCloudStorageToBigQueryOperator"
description: "Task to load CSV data to a BigQuery table"
args:
task_id: "load_inpatient_2011_to_bq"
bucket: "{{ var.value.composer_bucket }}"
source_objects: ["data/cms_medicare/inpatient_charges_2011/data_output.csv"]
source_format: "CSV"
destination_project_dataset_table: "cms_medicare.inpatient_charges_2011"
skip_leading_rows: 1
write_disposition: "WRITE_TRUNCATE"
schema_fields:
- description: "The CMS Certification Number (CCN) of the provider billing for outpatient hospital services"
name: "provider_id"
type: "STRING"
mode: "REQUIRED"
- description: "The name of the provider"
name: "provider_name"
type: "STRING"
mode: "NULLABLE"
- description: "The street address in which the provider is physically located"
name: "provider_street_address"
type: "STRING"
mode: "NULLABLE"
- description: "The city in which the provider is physically located"
name: "provider_city"
type: "STRING"
mode: "NULLABLE"
- description: "The state in which the provider is physically located"
name: "provider_state"
type: "STRING"
mode: "NULLABLE"
- description: "The zip code in which the provider is physically located"
name: "provider_zipcode"
type: "STRING"
mode: "NULLABLE"
- description: "The code and description identifying the MS-DRG. MS-DRGs are a classification system that groups similar clinical conditions (diagnoses) and the procedures furnished by the hospital during the stay"
name: "drg_definition"
type: "STRING"
mode: "REQUIRED"
- description: "The Hospital Referral Region (HRR) in which the provider is physically located"
name: "hospital_referral_region_description"
type: "STRING"
mode: "NULLABLE"
- description: "The number of discharges billed by the provider for inpatient hospital services"
name: "total_discharges"
type: "INTEGER"
mode: "NULLABLE"
- description: "The provider's average charge for services covered by Medicare for all discharges in the MS-DRG. These will vary from hospital to hospital because of differences in hospital charge structures"
name: "average_covered_charges"
type: "FLOAT"
mode: "NULLABLE"
- description: "The average total payments to all providers for the MS-DRG including the MSDRG amount, teaching, disproportionate share, capital, and outlier payments for all cases. Also included 5 in average total payments are co-payment and deductible amounts that the patient is responsible for and any additional payments by third parties for coordination of benefits"
name: "average_total_payments"
type: "FLOAT"
mode: "NULLABLE"
- description: "The average amount that Medicare pays to the provider for Medicare's share of the MS-DRG. Average Medicare payment amounts include the MS-DRG amount, teaching, disproportionate share, capital, and outlier payments for all cases. Medicare payments DO NOT include beneficiary co-payments and deductible amounts nor any additional payments from third parties for coordination of benefits"
name: "average_medicare_payments"
type: "FLOAT"
mode: "NULLABLE"
- operator: "GoogleCloudStorageToBigQueryOperator"
description: "Task to load CSV data to a BigQuery table"
args:
task_id: "load_inpatient_2012_to_bq"
bucket: "{{ var.value.composer_bucket }}"
source_objects: ["data/cms_medicare/inpatient_charges_2012/data_output.csv"]
source_format: "CSV"
destination_project_dataset_table: "cms_medicare.inpatient_charges_2012"
skip_leading_rows: 1
write_disposition: "WRITE_TRUNCATE"
schema_fields:
- description: "The CMS Certification Number (CCN) of the provider billing for outpatient hospital services"
name: "provider_id"
type: "STRING"
mode: "REQUIRED"
- description: "The name of the provider"
name: "provider_name"
type: "STRING"
mode: "NULLABLE"
- description: "The street address in which the provider is physically located"
name: "provider_street_address"
type: "STRING"
mode: "NULLABLE"
- description: "The city in which the provider is physically located"
name: "provider_city"
type: "STRING"
mode: "NULLABLE"
- description: "The state in which the provider is physically located"
name: "provider_state"
type: "STRING"
mode: "NULLABLE"
- description: "The zip code in which the provider is physically located"
name: "provider_zipcode"
type: "STRING"
mode: "NULLABLE"
- description: "The code and description identifying the MS-DRG. MS-DRGs are a classification system that groups similar clinical conditions (diagnoses) and the procedures furnished by the hospital during the stay"
name: "drg_definition"
type: "STRING"
mode: "REQUIRED"
- description: "The Hospital Referral Region (HRR) in which the provider is physically located"
name: "hospital_referral_region_description"
type: "STRING"
mode: "NULLABLE"
- description: "The number of discharges billed by the provider for inpatient hospital services"
name: "total_discharges"
type: "INTEGER"
mode: "NULLABLE"
- description: "The provider's average charge for services covered by Medicare for all discharges in the MS-DRG. These will vary from hospital to hospital because of differences in hospital charge structures"
name: "average_covered_charges"
type: "FLOAT"
mode: "NULLABLE"
- description: "The average total payments to all providers for the MS-DRG including the MSDRG amount, teaching, disproportionate share, capital, and outlier payments for all cases. Also included 5 in average total payments are co-payment and deductible amounts that the patient is responsible for and any additional payments by third parties for coordination of benefits"
name: "average_total_payments"
type: "FLOAT"
mode: "NULLABLE"
- description: "The average amount that Medicare pays to the provider for Medicare's share of the MS-DRG. Average Medicare payment amounts include the MS-DRG amount, teaching, disproportionate share, capital, and outlier payments for all cases. Medicare payments DO NOT include beneficiary co-payments and deductible amounts nor any additional payments from third parties for coordination of benefits"
name: "average_medicare_payments"
type: "FLOAT"
mode: "NULLABLE"
- operator: "GoogleCloudStorageToBigQueryOperator"
description: "Task to load CSV data to a BigQuery table"
args:
task_id: "load_inpatient_2013_to_bq"
bucket: "{{ var.value.composer_bucket }}"
source_objects: ["data/cms_medicare/inpatient_charges_2013/data_output.csv"]
source_format: "CSV"
destination_project_dataset_table: "cms_medicare.inpatient_charges_2013"
skip_leading_rows: 1
write_disposition: "WRITE_TRUNCATE"
schema_fields:
- description: "The CMS Certification Number (CCN) of the provider billing for outpatient hospital services"
name: "provider_id"
type: "STRING"
mode: "REQUIRED"
- description: "The name of the provider"
name: "provider_name"
type: "STRING"
mode: "NULLABLE"
- description: "The street address in which the provider is physically located"
name: "provider_street_address"
type: "STRING"
mode: "NULLABLE"
- description: "The city in which the provider is physically located"
name: "provider_city"
type: "STRING"
mode: "NULLABLE"
- description: "The state in which the provider is physically located"
name: "provider_state"
type: "STRING"
mode: "NULLABLE"
- description: "The zip code in which the provider is physically located"
name: "provider_zipcode"
type: "STRING"
mode: "NULLABLE"
- description: "The code and description identifying the MS-DRG. MS-DRGs are a classification system that groups similar clinical conditions (diagnoses) and the procedures furnished by the hospital during the stay"
name: "drg_definition"
type: "STRING"
mode: "REQUIRED"
- description: "The Hospital Referral Region (HRR) in which the provider is physically located"
name: "hospital_referral_region_description"
type: "STRING"
mode: "NULLABLE"
- description: "The number of discharges billed by the provider for inpatient hospital services"
name: "total_discharges"
type: "INTEGER"
mode: "NULLABLE"
- description: "The provider's average charge for services covered by Medicare for all discharges in the MS-DRG. These will vary from hospital to hospital because of differences in hospital charge structures"
name: "average_covered_charges"
type: "FLOAT"
mode: "NULLABLE"
- description: "The average total payments to all providers for the MS-DRG including the MSDRG amount, teaching, disproportionate share, capital, and outlier payments for all cases. Also included 5 in average total payments are co-payment and deductible amounts that the patient is responsible for and any additional payments by third parties for coordination of benefits"
name: "average_total_payments"
type: "FLOAT"
mode: "NULLABLE"
- description: "The average amount that Medicare pays to the provider for Medicare's share of the MS-DRG. Average Medicare payment amounts include the MS-DRG amount, teaching, disproportionate share, capital, and outlier payments for all cases. Medicare payments DO NOT include beneficiary co-payments and deductible amounts nor any additional payments from third parties for coordination of benefits"
name: "average_medicare_payments"
type: "FLOAT"
mode: "NULLABLE"
- operator: "GoogleCloudStorageToBigQueryOperator"
description: "Task to load CSV data to a BigQuery table"
args:
task_id: "load_inpatient_2014_to_bq"
bucket: "{{ var.value.composer_bucket }}"
source_objects: ["data/cms_medicare/inpatient_charges_2014/data_output.csv"]
source_format: "CSV"
destination_project_dataset_table: "cms_medicare.inpatient_charges_2014"
skip_leading_rows: 1
write_disposition: "WRITE_TRUNCATE"
schema_fields:
- description: "The CMS Certification Number (CCN) of the provider billing for outpatient hospital services"
name: "provider_id"
type: "STRING"
mode: "REQUIRED"
- description: "The name of the provider"
name: "provider_name"
type: "STRING"
mode: "NULLABLE"
- description: "The street address in which the provider is physically located"
name: "provider_street_address"
type: "STRING"
mode: "NULLABLE"
- description: "The city in which the provider is physically located"
name: "provider_city"
type: "STRING"
mode: "NULLABLE"
- description: "The state in which the provider is physically located"
name: "provider_state"
type: "STRING"
mode: "NULLABLE"
- description: "The zip code in which the provider is physically located"
name: "provider_zipcode"
type: "STRING"
mode: "NULLABLE"
- description: "The code and description identifying the MS-DRG. MS-DRGs are a classification system that groups similar clinical conditions (diagnoses) and the procedures furnished by the hospital during the stay"
name: "drg_definition"
type: "STRING"
mode: "REQUIRED"
- description: "The Hospital Referral Region (HRR) in which the provider is physically located"
name: "hospital_referral_region_description"
type: "STRING"
mode: "NULLABLE"
- description: "The number of discharges billed by the provider for inpatient hospital services"
name: "total_discharges"
type: "INTEGER"
mode: "NULLABLE"
- description: "The provider's average charge for services covered by Medicare for all discharges in the MS-DRG. These will vary from hospital to hospital because of differences in hospital charge structures"
name: "average_covered_charges"
type: "FLOAT"
mode: "NULLABLE"
- description: "The average total payments to all providers for the MS-DRG including the MSDRG amount, teaching, disproportionate share, capital, and outlier payments for all cases. Also included 5 in average total payments are co-payment and deductible amounts that the patient is responsible for and any additional payments by third parties for coordination of benefits"
name: "average_total_payments"
type: "FLOAT"
mode: "NULLABLE"
- description: "The average amount that Medicare pays to the provider for Medicare's share of the MS-DRG. Average Medicare payment amounts include the MS-DRG amount, teaching, disproportionate share, capital, and outlier payments for all cases. Medicare payments DO NOT include beneficiary co-payments and deductible amounts nor any additional payments from third parties for coordination of benefits"
name: "average_medicare_payments"
type: "FLOAT"
mode: "NULLABLE"
- operator: "GoogleCloudStorageToBigQueryOperator"
description: "Task to load CSV data to a BigQuery table"
args:
task_id: "load_inpatient_2015_to_bq"
bucket: "{{ var.value.composer_bucket }}"
source_objects: ["data/cms_medicare/inpatient_charges_2015/data_output.csv"]
source_format: "CSV"
destination_project_dataset_table: "cms_medicare.inpatient_charges_2015"
skip_leading_rows: 1
write_disposition: "WRITE_TRUNCATE"
schema_fields:
- description: "The CMS Certification Number (CCN) of the provider billing for outpatient hospital services"
name: "provider_id"
type: "STRING"
mode: "REQUIRED"
- description: "The name of the provider"
name: "provider_name"
type: "STRING"
mode: "NULLABLE"
- description: "The street address in which the provider is physically located"
name: "provider_street_address"
type: "STRING"
mode: "NULLABLE"
- description: "The city in which the provider is physically located"
name: "provider_city"
type: "STRING"
mode: "NULLABLE"
- description: "The state in which the provider is physically located"
name: "provider_state"
type: "STRING"
mode: "NULLABLE"
- description: "The zip code in which the provider is physically located"
name: "provider_zipcode"
type: "STRING"
mode: "NULLABLE"
- description: "The code and description identifying the MS-DRG. MS-DRGs are a classification system that groups similar clinical conditions (diagnoses) and the procedures furnished by the hospital during the stay"
name: "drg_definition"
type: "STRING"
mode: "REQUIRED"
- description: "The Hospital Referral Region (HRR) in which the provider is physically located"
name: "hospital_referral_region_description"
type: "STRING"
mode: "NULLABLE"
- description: "The number of discharges billed by the provider for inpatient hospital services"
name: "total_discharges"
type: "INTEGER"
mode: "NULLABLE"
- description: "The provider's average charge for services covered by Medicare for all discharges in the MS-DRG. These will vary from hospital to hospital because of differences in hospital charge structures"
name: "average_covered_charges"
type: "FLOAT"
mode: "NULLABLE"
- description: "The average total payments to all providers for the MS-DRG including the MSDRG amount, teaching, disproportionate share, capital, and outlier payments for all cases. Also included 5 in average total payments are co-payment and deductible amounts that the patient is responsible for and any additional payments by third parties for coordination of benefits"
name: "average_total_payments"
type: "FLOAT"
mode: "NULLABLE"
- description: "The average amount that Medicare pays to the provider for Medicare's share of the MS-DRG. Average Medicare payment amounts include the MS-DRG amount, teaching, disproportionate share, capital, and outlier payments for all cases. Medicare payments DO NOT include beneficiary co-payments and deductible amounts nor any additional payments from third parties for coordination of benefits"
name: "average_medicare_payments"
type: "FLOAT"
mode: "NULLABLE"
graph_paths:
- "inpatient_2011_transform_csv >> load_inpatient_2011_to_bq"
- "inpatient_2012_transform_csv >> load_inpatient_2012_to_bq"
- "inpatient_2013_transform_csv >> load_inpatient_2013_to_bq"
- "inpatient_2014_transform_csv >> load_inpatient_2014_to_bq"
- "inpatient_2015_transform_csv >> load_inpatient_2015_to_bq"