Book a Demo!
CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutPoliciesSign UpSign In
CloudPak-Outcomes
GitHub Repository: CloudPak-Outcomes/Outcomes-Projects
Path: blob/main/Data-Product-Hub-L3/import.py
1928 views
1
import time
2
from client import ImportClient
3
4
client = ImportClient()
5
6
7
# Check Bearer Token is valid
8
bearer_token = client.get_bearer_token()
9
if bearer_token:
10
print("")
11
print(f"Bearer token Aquired")
12
print("")
13
14
# Cloud Object Storage
15
client.define_cos_connection(bearer_token, catalog=False)
16
17
# DB2 Warehouse
18
client.define_db2_connection(bearer_token, catalog=False)
19
20
# PostgreSQL Database
21
client.define_psql_connection(bearer_token, catalog=False)
22
23
print("")
24
# Import Governance Artifacts
25
#client.main_import_process("governance_artifacts.zip", process_id=None)
26
print("")
27
28
# Create and run the metadata import DB2 Warehouse
29
print("Running Metadata Import for DB2 Warehouse")
30
path_db2=["/EMPLOYEE/EMPLOYEE_HISTORY","/EMPLOYEE/EMPLOYEE_RECORDS","/EMPLOYEE/EMPLOYEE_SUMMARY","/EMPLOYEE/EMPLOYEE"]
31
db2_mdid, db2_mdi_response = client.create_and_run_metadata_import(client.db2_id, path_db2, name="2 DB2 Metadata Import")
32
33
# Create and run the metadata import COS
34
print("Running Metadata Import for Cloud Object Storage")
35
path_cos = ["/cpd-outcomes/Warehouse/WAREHOUSE_ASSIGNED_SHIFTS.csv","/cpd-outcomes/Warehouse/WAREHOUSE_SHIFTS.csv","/cpd-outcomes/Warehouse/WAREHOUSE_STAFF.csv","/cpd-outcomes/Warehouse/WAREHOUSE_STAFFING.csv"]
36
cos_mdid, cos_mdi_response = client.create_and_run_metadata_import(client.cos_id, path_cos, name="2 Cloud Object Storage Metadata Import")
37
38
# Create and run the metadata import
39
print("Running Metadata Import for PostgreSQL")
40
path_psql = ["/CUSTOMER/CUSTOMER_LOYALTY"]
41
psql_mdid, psql_mdid_response = client.create_and_run_metadata_import(client.psql_id, path_psql, name="2 Postgresql Metadata Import")
42
43
# Metadata Enrichment
44
print("Running Metadata Enrichment for DB2 Warehouse")
45
db2_result = client.create_and_run_metadata_enrichment(
46
name="Db2 Warehouse MDE",
47
mdi_id=db2_mdid,
48
job_name=client.db2_name+" Enrichment Job",
49
publish_job_name=client.db2_name+" publish Job",
50
)
51
52
print("🌩️ Running Metadata Enrichment for Cloud Object Storage 🌩️")
53
cos_result = client.create_and_run_metadata_enrichment(
54
name="Cloud Object Storage Enrichment",
55
mdi_id=cos_mdid,
56
job_name=client.cos_name+" Enrichment Job",
57
publish_job_name=client.cos_name+" publish Job",
58
)
59
60
print("🐘 Running Metadata Enrichment for PostgreSQL 🐘")
61
psql_result = client.create_and_run_metadata_enrichment(
62
name="Postgresql MDE",
63
mdi_id=psql_mdid,
64
job_name=client.psql_name+" Enrichment Job",
65
publish_job_name=client.psql_name+" publish Job",
66
)
67
68
69
print("Sleeping for 60 seconds to allow enrichment to complete")
70
time.sleep(60)
71
print("Sleeping for 60 seconds to allow enrichment to complete")
72
time.sleep(60)
73
print("Sleeping for 60 seconds to allow enrichment to complete")
74
time.sleep(60)
75
# Publish Assets
76
print("")
77
db2_mde_id = db2_result.get("metadata", {}).get("asset_id")
78
db2_publish_result = client.publish_metadata_enrichment_assets(db2_mde_id)
79
print("DB2 Warehouse Successfully Published")
80
81
cos_mde_id = cos_result.get("metadata", {}).get("asset_id")
82
cos_publish_result = client.publish_metadata_enrichment_assets(cos_mde_id)
83
print("Cloud Object Storage Successfully Published")
84
85
psql_mde_id = psql_result.get("metadata", {}).get("asset_id")
86
psql_publish_result = client.publish_metadata_enrichment_assets(psql_mde_id)
87
print("PostgreSQL Successfully Published")
88
89
print("Congratulations! You have successfully setup the Data Product Hub Environment.")
90
91