Path: blob/main/Data-Product-Hub-L3/import.py
1928 views
import time1from client import ImportClient23client = ImportClient()456# Check Bearer Token is valid7bearer_token = client.get_bearer_token()8if bearer_token:9print("")10print(f"Bearer token Aquired")11print("")1213# Cloud Object Storage14client.define_cos_connection(bearer_token, catalog=False)1516# DB2 Warehouse17client.define_db2_connection(bearer_token, catalog=False)1819# PostgreSQL Database20client.define_psql_connection(bearer_token, catalog=False)2122print("")23# Import Governance Artifacts24#client.main_import_process("governance_artifacts.zip", process_id=None)25print("")2627# Create and run the metadata import DB2 Warehouse28print("Running Metadata Import for DB2 Warehouse")29path_db2=["/EMPLOYEE/EMPLOYEE_HISTORY","/EMPLOYEE/EMPLOYEE_RECORDS","/EMPLOYEE/EMPLOYEE_SUMMARY","/EMPLOYEE/EMPLOYEE"]30db2_mdid, db2_mdi_response = client.create_and_run_metadata_import(client.db2_id, path_db2, name="2 DB2 Metadata Import")3132# Create and run the metadata import COS33print("Running Metadata Import for Cloud Object Storage")34path_cos = ["/cpd-outcomes/Warehouse/WAREHOUSE_ASSIGNED_SHIFTS.csv","/cpd-outcomes/Warehouse/WAREHOUSE_SHIFTS.csv","/cpd-outcomes/Warehouse/WAREHOUSE_STAFF.csv","/cpd-outcomes/Warehouse/WAREHOUSE_STAFFING.csv"]35cos_mdid, cos_mdi_response = client.create_and_run_metadata_import(client.cos_id, path_cos, name="2 Cloud Object Storage Metadata Import")3637# Create and run the metadata import38print("Running Metadata Import for PostgreSQL")39path_psql = ["/CUSTOMER/CUSTOMER_LOYALTY"]40psql_mdid, psql_mdid_response = client.create_and_run_metadata_import(client.psql_id, path_psql, name="2 Postgresql Metadata Import")4142# Metadata Enrichment43print("Running Metadata Enrichment for DB2 Warehouse")44db2_result = client.create_and_run_metadata_enrichment(45name="Db2 Warehouse MDE",46mdi_id=db2_mdid,47job_name=client.db2_name+" Enrichment Job",48publish_job_name=client.db2_name+" publish Job",49)5051print("🌩️ Running Metadata Enrichment for Cloud Object Storage 🌩️")52cos_result = client.create_and_run_metadata_enrichment(53name="Cloud Object Storage Enrichment",54mdi_id=cos_mdid,55job_name=client.cos_name+" Enrichment Job",56publish_job_name=client.cos_name+" publish Job",57)5859print("🐘 Running Metadata Enrichment for PostgreSQL 🐘")60psql_result = client.create_and_run_metadata_enrichment(61name="Postgresql MDE",62mdi_id=psql_mdid,63job_name=client.psql_name+" Enrichment Job",64publish_job_name=client.psql_name+" publish Job",65)666768print("Sleeping for 60 seconds to allow enrichment to complete")69time.sleep(60)70print("Sleeping for 60 seconds to allow enrichment to complete")71time.sleep(60)72print("Sleeping for 60 seconds to allow enrichment to complete")73time.sleep(60)74# Publish Assets75print("")76db2_mde_id = db2_result.get("metadata", {}).get("asset_id")77db2_publish_result = client.publish_metadata_enrichment_assets(db2_mde_id)78print("DB2 Warehouse Successfully Published")7980cos_mde_id = cos_result.get("metadata", {}).get("asset_id")81cos_publish_result = client.publish_metadata_enrichment_assets(cos_mde_id)82print("Cloud Object Storage Successfully Published")8384psql_mde_id = psql_result.get("metadata", {}).get("asset_id")85psql_publish_result = client.publish_metadata_enrichment_assets(psql_mde_id)86print("PostgreSQL Successfully Published")8788print("Congratulations! You have successfully setup the Data Product Hub Environment.")899091