From aeac4aa70a029fbefc354995c3abf50be1b300de Mon Sep 17 00:00:00 2001 From: "rushabh.mehta" Date: Wed, 18 Dec 2024 13:48:22 +0530 Subject: [PATCH 1/6] dashboard deploy from dev to target workspace --- knowledge_base/dashboard_deploy/.gitignore | 2 + knowledge_base/dashboard_deploy/README.md | 46 ++++++++++++++++++ .../dashboard_deploy/dashboard_deploy.sh | 47 +++++++++++++++++++ .../dashboard_json_query_modify.py | 42 +++++++++++++++++ .../dashboard_deploy/databricks.yml | 28 +++++++++++ .../dashboard_deploy/resources/dashboard.yml | 25 ++++++++++ 6 files changed, 190 insertions(+) create mode 100644 knowledge_base/dashboard_deploy/.gitignore create mode 100644 knowledge_base/dashboard_deploy/README.md create mode 100644 knowledge_base/dashboard_deploy/dashboard_deploy.sh create mode 100644 knowledge_base/dashboard_deploy/dashboard_json_query_modify.py create mode 100644 knowledge_base/dashboard_deploy/databricks.yml create mode 100644 knowledge_base/dashboard_deploy/resources/dashboard.yml diff --git a/knowledge_base/dashboard_deploy/.gitignore b/knowledge_base/dashboard_deploy/.gitignore new file mode 100644 index 0000000..de811f1 --- /dev/null +++ b/knowledge_base/dashboard_deploy/.gitignore @@ -0,0 +1,2 @@ + +.databricks diff --git a/knowledge_base/dashboard_deploy/README.md b/knowledge_base/dashboard_deploy/README.md new file mode 100644 index 0000000..7c7ff74 --- /dev/null +++ b/knowledge_base/dashboard_deploy/README.md @@ -0,0 +1,46 @@ +# Dashboard for NYC Taxi Trip Analysis + +This example demonstrates how to define an AI/BI dashboard in a Databricks Asset Bundle. + +It includes and deploys the sample __NYC Taxi Trip Analysis__ dashboard to a Databricks workspace. + +For more information about AI/BI dashboards, please refer to the [documentation](https://docs.databricks.com/dashboards/index.html). + +## Prerequisites + +* Databricks CLI v0.232.0 or above + +## Usage + +Modify `databricks.yml`: +* Update the `host` field under `workspace` to the Databricks workspace to deploy to. +* Update the `warehouse` field under `warehouse_id` to the name of the SQL warehouse to use. + +Run `databricks bundle deploy` to deploy the dashboard. + +Run `databricks bundle open` to navigate to the deployed dashboard in your browser. Alternatively, run `databricks bundle summary` to display its URL. + +### Visual modification + +You can use the Databricks UI to modify the dashboard, but any modifications made through the UI will not be applied to the bundle `.lvdash.json` file unless you explicitly update it. + +To update the local bundle `.lvdash.json` file, run: + +```sh +databricks bundle generate dashboard --resource nyc_taxi_trip_analysis --force +``` + +To continuously poll and retrieve the updated `.lvdash.json` file when it changes, run: + +```sh +databricks bundle generate dashboard --resource nyc_taxi_trip_analysis --force --watch +``` + +Any remote modifications of a dashboard are noticed by the `deploy` command and require +you to acknowledge that remote changes can be overwritten by local changes. +It is therefore recommended to run the `generate` command before running the `deploy` command. +Otherwise, you may lose your remote changes. + +### Manual modification + +You can modify the `.lvdash.json` file directly and redeploy to observe your changes. diff --git a/knowledge_base/dashboard_deploy/dashboard_deploy.sh b/knowledge_base/dashboard_deploy/dashboard_deploy.sh new file mode 100644 index 0000000..c124976 --- /dev/null +++ b/knowledge_base/dashboard_deploy/dashboard_deploy.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Check if the correct number of arguments is provided +if [ "$#" -ne 5 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Assign arguments to variables +dashboard_id=$1 +dev_workspace=$2 +target_workspace=$3 +target_warehouse_name=$4 +replacements=$5 +access_group=$6 +dashboard_name=$7 + +# Print the received arguments +echo "Dashboard ID: $dashboard_id" +echo "Development Workspace: $dev_workspace" +echo "Target Workspace: $target_workspace" +echo "Target Warehouse Name: $target_warehouse_name" +echo "Replacements: $replacements" +echo "dashboard_name: $dashboard_name" + + +echo "Running script with the provided arguments..." + +rm dev_dashboard.json +echo "============================================" +echo "Get the Dev Dashboard metadata json" +databricks lakeview get $dashboard_id --output json --profile dev >> dev_dashboard.json + +echo "============================================" +echo "modify Query in Dashboard metadata json of catalog and schema names of target workspace" +rm ./src/dashboard.json +python3 dashboard_json_query_modify.py --file_path dev_dashboard.json --replacements "$replacements" --target_file ./src/dashboard.json + +echo "============================================" +echo "Validate DAB bundle dashboard to the target workspace" +databricks bundle validate --var "dev_workspace=$dev_workspace" --var "target_workspace=${target_workspace}" \ +--var "warehouse_name=${target_warehouse_name}" --var "access_group=${access_group}" --var "dashboard_name=${dashboard_name}" --profile target + +echo "============================================" +echo "Deploy the modified dashboard to the target workspace" +databricks bundle deploy --var "dev_workspace=${dev_workspace}" --var "target_workspace=${target_workspace}" \ +--var "warehouse_name=${target_warehouse_name}" --var "access_group=${access_group}" --var "dashboard_name=${dashboard_name}" --profile target \ No newline at end of file diff --git a/knowledge_base/dashboard_deploy/dashboard_json_query_modify.py b/knowledge_base/dashboard_deploy/dashboard_json_query_modify.py new file mode 100644 index 0000000..f4e3093 --- /dev/null +++ b/knowledge_base/dashboard_deploy/dashboard_json_query_modify.py @@ -0,0 +1,42 @@ +import json +import argparse +def replace_keys_in_query(query_string, replacements): + for key, value in replacements.items(): + query_string = query_string.replace(key, value) + return query_string + +def query_in_datasets(data, replacements): + if isinstance(data, dict): + if 'datasets' in data: + for dataset in data['datasets']: + if 'query' in dataset: + dataset['query'] = replace_keys_in_query(dataset['query'], replacements) + + return data + +def parse_json_file(file_path, replacements, target_file_path): + with open(file_path, 'r') as file: + data = json.load(file) + + dashboard_json = json.loads(data['serialized_dashboard']) + modified_data = query_in_datasets(dashboard_json, replacements) + #target_file_path ="./src/dashboard.json" + with open(target_file_path, 'w') as file: + json.dump(modified_data, file, indent=4) + +# Example usage +#replacements = '{\"rushabh_mehta_catalog\": \"rushabh_mehta_catalog_new\", \"ru_nyctaxi\": \"ru_nyctaxi_new\"}' + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Parse JSON file') + + parser.add_argument('--file_path', required=True, help='Path to the Dashboard JSON file') + parser.add_argument('--replacements', required=True, help='Replacements in JSON format') + parser.add_argument('--target_file', required=True, help='Dashboard json modified file save location') + + + args = parser.parse_args() + print(args) + print(args.replacements) + + parse_json_file(args.file_path, json.loads(f'{args.replacements}'), args.target_file) \ No newline at end of file diff --git a/knowledge_base/dashboard_deploy/databricks.yml b/knowledge_base/dashboard_deploy/databricks.yml new file mode 100644 index 0000000..e27a2c4 --- /dev/null +++ b/knowledge_base/dashboard_deploy/databricks.yml @@ -0,0 +1,28 @@ +bundle: + name: ${var.dashboard_name} + +include: + - resources/*.yml + +variables: + # The "warehouse_id" variable is used to reference the warehouse used by the dashboard. + warehouse_id: + lookup: + # Replace this with the name of your SQL warehouse. + warehouse: ${var.warehouse_name} + dev_workspace: + default: "host" + target_workspace: + default: "target_workspace" + dashboard_name: + default: "def" + access_group: + default: "def group" + warehouse_name: + default: "warehouse" + + +targets: + target: + default: true + diff --git a/knowledge_base/dashboard_deploy/resources/dashboard.yml b/knowledge_base/dashboard_deploy/resources/dashboard.yml new file mode 100644 index 0000000..91c154c --- /dev/null +++ b/knowledge_base/dashboard_deploy/resources/dashboard.yml @@ -0,0 +1,25 @@ +resources: + dashboards: + dashboard: + display_name: ${var.dashboard_name} + file_path: ../src/dashboard.json + warehouse_id: ${var.warehouse_id} + + # If "embed_credentials" is set to true, the bundle deployment identity's credentials + # are used to execute queries for all dashboard viewers. + # If it is set to false, each viewer's own credentials will be used instead. + # + # It defaults to false. + # + # embed_credentials: false + + # The "parent_path" field can be configured to place the dashboard in a + # non-standard folder in the workspace. + # + # It defaults to "${workspace.resource_path}", which is located + # under the bundle deployment root. + # + # parent_path: ${workspace.resource_path} + permissions: + - level: CAN_READ + group_name: ${var.access_group} From 07ff59002ae4608548795525475f3aa5162ad427 Mon Sep 17 00:00:00 2001 From: "rushabh.mehta" Date: Wed, 18 Dec 2024 15:13:02 +0530 Subject: [PATCH 2/6] dashboard deploy from dev to target workspace --- knowledge_base/dashboard_deploy/README.md | 45 +++++-------- .../dashboard_deploy/dashboard_deploy.sh | 63 ++++++++++++------- .../dashboard_json_query_modify.py | 5 +- 3 files changed, 57 insertions(+), 56 deletions(-) diff --git a/knowledge_base/dashboard_deploy/README.md b/knowledge_base/dashboard_deploy/README.md index 7c7ff74..fd8f233 100644 --- a/knowledge_base/dashboard_deploy/README.md +++ b/knowledge_base/dashboard_deploy/README.md @@ -1,46 +1,29 @@ -# Dashboard for NYC Taxi Trip Analysis +# Dashboard Deployment to higher environment -This example demonstrates how to define an AI/BI dashboard in a Databricks Asset Bundle. - -It includes and deploys the sample __NYC Taxi Trip Analysis__ dashboard to a Databricks workspace. +This example demonstrates how to deploy the existing dashboard from dev to target Databricks workspace, using dab bundle +it also allows you to change the catalog and schema names of dashboard json dataset query from dev to target +specifies access to the groups on the target dashboard For more information about AI/BI dashboards, please refer to the [documentation](https://docs.databricks.com/dashboards/index.html). ## Prerequisites * Databricks CLI v0.232.0 or above +* Databricks configuration file (`~/.databricks.cfg`) with a profile named dev and target workspace configured +* The catalog, schema and tables mentioned in dashboard datasets exists in target workspace -## Usage - -Modify `databricks.yml`: -* Update the `host` field under `workspace` to the Databricks workspace to deploy to. -* Update the `warehouse` field under `warehouse_id` to the name of the SQL warehouse to use. - -Run `databricks bundle deploy` to deploy the dashboard. - -Run `databricks bundle open` to navigate to the deployed dashboard in your browser. Alternatively, run `databricks bundle summary` to display its URL. -### Visual modification - -You can use the Databricks UI to modify the dashboard, but any modifications made through the UI will not be applied to the bundle `.lvdash.json` file unless you explicitly update it. - -To update the local bundle `.lvdash.json` file, run: - -```sh -databricks bundle generate dashboard --resource nyc_taxi_trip_analysis --force -``` +## Usage -To continuously poll and retrieve the updated `.lvdash.json` file when it changes, run: +Invoke the shell script dashboard_deploy.sh to deploy the dashboard from dev to target Databricks workspace. -```sh -databricks bundle generate dashboard --resource nyc_taxi_trip_analysis --force --watch +Example +``` sh + # 'dev-dashboard-id' 'target-warehouse 'dev to target catalog/schema mapping' 'access 'target dashboard name' + # name' group' + +sh dashboard_deploy.sh 01efbbade84416dc984927ca794fd768 'Shared Endpoint' '{"samples": "shared", "nyctaxi": "ru_nyctaxi"}' test_group client_dashboard ``` -Any remote modifications of a dashboard are noticed by the `deploy` command and require -you to acknowledge that remote changes can be overwritten by local changes. -It is therefore recommended to run the `generate` command before running the `deploy` command. -Otherwise, you may lose your remote changes. -### Manual modification -You can modify the `.lvdash.json` file directly and redeploy to observe your changes. diff --git a/knowledge_base/dashboard_deploy/dashboard_deploy.sh b/knowledge_base/dashboard_deploy/dashboard_deploy.sh index c124976..c43ef35 100644 --- a/knowledge_base/dashboard_deploy/dashboard_deploy.sh +++ b/knowledge_base/dashboard_deploy/dashboard_deploy.sh @@ -2,46 +2,67 @@ # Check if the correct number of arguments is provided if [ "$#" -ne 5 ]; then - echo "Usage: $0 " + echo "Usage: $0 " exit 1 fi # Assign arguments to variables dashboard_id=$1 -dev_workspace=$2 -target_workspace=$3 -target_warehouse_name=$4 -replacements=$5 -access_group=$6 -dashboard_name=$7 +target_warehouse_name=$2 +replacements=$3 +access_group=$4 +dashboard_name=$5 + +# Function to check the result of the last command and exit if it failed +check_result() { + if [ $1 -ne 0 ]; then + echo "Error: $2" + exit 1 + fi +} + +clean_up() { + rm -rf .databricks + rm -f dev_dashboard.json + rm -f ./src/dashboard.json +} # Print the received arguments echo "Dashboard ID: $dashboard_id" -echo "Development Workspace: $dev_workspace" -echo "Target Workspace: $target_workspace" echo "Target Warehouse Name: $target_warehouse_name" echo "Replacements: $replacements" echo "dashboard_name: $dashboard_name" - -echo "Running script with the provided arguments..." - -rm dev_dashboard.json echo "============================================" +echo "Clean up the temporary files if any" +clean_up +check_result $? "Failed to clean up the temporary files" +echo "********************************************" +echo "============================================" +echo "Running script with the provided arguments..." echo "Get the Dev Dashboard metadata json" databricks lakeview get $dashboard_id --output json --profile dev >> dev_dashboard.json - +check_result $? "Failed to get the dashboard metadata json" +echo "********************************************" echo "============================================" echo "modify Query in Dashboard metadata json of catalog and schema names of target workspace" -rm ./src/dashboard.json python3 dashboard_json_query_modify.py --file_path dev_dashboard.json --replacements "$replacements" --target_file ./src/dashboard.json - +check_result $? "Failed to modify the dashboard metadata json" +echo "********************************************" echo "============================================" echo "Validate DAB bundle dashboard to the target workspace" -databricks bundle validate --var "dev_workspace=$dev_workspace" --var "target_workspace=${target_workspace}" \ ---var "warehouse_name=${target_warehouse_name}" --var "access_group=${access_group}" --var "dashboard_name=${dashboard_name}" --profile target - +databricks bundle validate --var "warehouse_name=${target_warehouse_name}" --var "access_group=${access_group}" \ +--var "dashboard_name=${dashboard_name}" --profile target +check_result $? "Failed to validate the dashboard bundle" +echo "********************************************" echo "============================================" echo "Deploy the modified dashboard to the target workspace" -databricks bundle deploy --var "dev_workspace=${dev_workspace}" --var "target_workspace=${target_workspace}" \ ---var "warehouse_name=${target_warehouse_name}" --var "access_group=${access_group}" --var "dashboard_name=${dashboard_name}" --profile target \ No newline at end of file +databricks bundle deploy --var "warehouse_name=${target_warehouse_name}" --var "access_group=${access_group}" \ +--var "dashboard_name=${dashboard_name}" --profile target +check_result $? "Failed to deploy the dashboard bundle" +echo "********************************************" +echo "============================================" +echo "Optional-Clean up the temporary files" +clean_up +check_result $? "Failed to clean up the temporary files" +echo "********************************************" \ No newline at end of file diff --git a/knowledge_base/dashboard_deploy/dashboard_json_query_modify.py b/knowledge_base/dashboard_deploy/dashboard_json_query_modify.py index f4e3093..288e077 100644 --- a/knowledge_base/dashboard_deploy/dashboard_json_query_modify.py +++ b/knowledge_base/dashboard_deploy/dashboard_json_query_modify.py @@ -20,13 +20,10 @@ def parse_json_file(file_path, replacements, target_file_path): dashboard_json = json.loads(data['serialized_dashboard']) modified_data = query_in_datasets(dashboard_json, replacements) - #target_file_path ="./src/dashboard.json" + with open(target_file_path, 'w') as file: json.dump(modified_data, file, indent=4) -# Example usage -#replacements = '{\"rushabh_mehta_catalog\": \"rushabh_mehta_catalog_new\", \"ru_nyctaxi\": \"ru_nyctaxi_new\"}' - if __name__ == "__main__": parser = argparse.ArgumentParser(description='Parse JSON file') From 76b894767c1d30ac212b81c561796e67f79f7fe6 Mon Sep 17 00:00:00 2001 From: "rushabh.mehta" Date: Wed, 18 Dec 2024 15:24:05 +0530 Subject: [PATCH 3/6] dashboard deploy from dev to target workspace --- knowledge_base/dashboard_deploy/src/temp | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 knowledge_base/dashboard_deploy/src/temp diff --git a/knowledge_base/dashboard_deploy/src/temp b/knowledge_base/dashboard_deploy/src/temp new file mode 100644 index 0000000..e69de29 From a562f1fa1481668e7a49b8732b8cd85ff2c9c3e8 Mon Sep 17 00:00:00 2001 From: "rushabh.mehta" Date: Wed, 18 Dec 2024 15:25:03 +0530 Subject: [PATCH 4/6] dashboard deploy from dev to target workspace --- knowledge_base/dashboard_deploy/src/temp | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 knowledge_base/dashboard_deploy/src/temp diff --git a/knowledge_base/dashboard_deploy/src/temp b/knowledge_base/dashboard_deploy/src/temp deleted file mode 100644 index e69de29..0000000 From 34cf096c867e1d011f30220599878fccebc489d6 Mon Sep 17 00:00:00 2001 From: "rushabh.mehta" Date: Wed, 18 Dec 2024 15:26:36 +0530 Subject: [PATCH 5/6] dashboard deploy from dev to target workspace --- knowledge_base/dashboard_deploy/src/temp | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 knowledge_base/dashboard_deploy/src/temp diff --git a/knowledge_base/dashboard_deploy/src/temp b/knowledge_base/dashboard_deploy/src/temp new file mode 100644 index 0000000..e69de29 From 2805c09c33c7c7b6db922e8717a2cbe7f3f6fc54 Mon Sep 17 00:00:00 2001 From: "rushabh.mehta" Date: Wed, 18 Dec 2024 15:29:40 +0530 Subject: [PATCH 6/6] dashboard deploy from dev to target workspace --- knowledge_base/dashboard_deploy/README.md | 34 +++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/knowledge_base/dashboard_deploy/README.md b/knowledge_base/dashboard_deploy/README.md index fd8f233..6068b03 100644 --- a/knowledge_base/dashboard_deploy/README.md +++ b/knowledge_base/dashboard_deploy/README.md @@ -24,6 +24,36 @@ Example sh dashboard_deploy.sh 01efbbade84416dc984927ca794fd768 'Shared Endpoint' '{"samples": "shared", "nyctaxi": "ru_nyctaxi"}' test_group client_dashboard ``` - - +Output + +Clean up the temporary files if any +******************************************** +============================================ +Running script with the provided arguments... +Get the Dev Dashboard metadata json +******************************************** +============================================ +modify Query in Dashboard metadata json of catalog and schema names of target workspace +Namespace(file_path='dev_dashboard.json', replacements='{"samples": "shared", "nyctaxi": "ru_nyctaxi"}', target_file='./src/dashboard.json') +{"samples": "shared", "nyctaxi": "ru_nyctaxi"} +******************************************** +============================================ +Validate DAB bundle dashboard to the target workspace +Name: client_dashboard +Target: target +Workspace: +User: rushabh.mehta@databricks.com +Path: /Workspace/Users/rushabh.mehta@databricks.com/.bundle/client_dashboard/target + +Validation OK! +******************************************** +============================================ +Deploy the modified dashboard to the target workspace +Uploading bundle files to /Workspace/Users/rushabh.mehta@databricks.com/.bundle/client_dashboard/target/files... +Deploying resources... +Updating deployment state... +Deployment complete! +******************************************** +============================================ +Optional-Clean up the temporary files