From 9536054bae3769fa91b02a7632456be78b3d4576 Mon Sep 17 00:00:00 2001 From: Liran Bareket Date: Mon, 16 Nov 2026 13:25:20 -0500 Subject: [PATCH] Add the UCX Databricks App and the deploy-app command A Databricks App to run the assessment, review and edit the table mapping, and migrate tables from a web UI, deployed with `databricks labs ucx deploy-app`. - Backend (FastAPI) in databricks.labs.ucx.app, reusing the CLI's code paths - Frontend (Next.js static export) in app-ui/, built into the wheel - On-behalf-of-user access, with the app's service principal for the Jobs API - Selected migration workflows run sequentially as one chained job run - deploy-app deploys the released wheel as an asset bundle - CI job that builds the frontend and checks the wheel contains the app Co-authored-by: Isaac --- .github/workflows/push.yml | 54 + .gitignore | 4 + Makefile | 19 +- README.md | 3 + app-ui/.gitignore | 5 + app-ui/app/assessment/page.tsx | 93 + app-ui/app/globals.css | 31 + app-ui/app/layout.tsx | 21 + app-ui/app/mapping/page.tsx | 240 +++ app-ui/app/migration/page.tsx | 138 ++ app-ui/app/page.tsx | 87 + app-ui/components/runs.tsx | 191 ++ app-ui/components/shell.tsx | 104 + app-ui/components/ui/alert.tsx | 32 + app-ui/components/ui/badge.tsx | 41 + app-ui/components/ui/button.tsx | 31 + app-ui/components/ui/card.tsx | 16 + app-ui/components/ui/checkbox.tsx | 25 + app-ui/components/ui/confirm-dialog.tsx | 50 + app-ui/components/ui/input.tsx | 16 + app-ui/next.config.ts | 19 + app-ui/package.json | 38 + app-ui/pnpm-lock.yaml | 1763 +++++++++++++++++ app-ui/postcss.config.mjs | 3 + app-ui/tsconfig.json | 42 + docs/ucx/docs/dev/contributing.mdx | 24 + docs/ucx/docs/reference/app.mdx | 101 + docs/ucx/docs/reference/commands/index.mdx | 13 + docs/ucx/docs/reference/index.mdx | 1 + labs.yml | 12 + pyproject.toml | 11 + src/databricks/labs/ucx/app/__init__.py | 1 + src/databricks/labs/ucx/app/auth.py | 63 + src/databricks/labs/ucx/app/bundle/app.yaml | 20 + .../labs/ucx/app/bundle/databricks.yml | 45 + src/databricks/labs/ucx/app/context.py | 43 + src/databricks/labs/ucx/app/deploy.py | 207 ++ src/databricks/labs/ucx/app/main.py | 48 + src/databricks/labs/ucx/app/models.py | 103 + .../labs/ucx/app/routes/__init__.py | 12 + .../labs/ucx/app/routes/assessment.py | 17 + .../labs/ucx/app/routes/dependencies.py | 21 + src/databricks/labs/ucx/app/routes/mapping.py | 22 + .../labs/ucx/app/routes/migration.py | 17 + src/databricks/labs/ucx/app/routes/runs.py | 18 + src/databricks/labs/ucx/app/routes/status.py | 11 + src/databricks/labs/ucx/app/service.py | 244 +++ src/databricks/labs/ucx/cli.py | 18 + .../labs/ucx/source_code/known.json | 50 + tests/integration/app/__init__.py | 0 tests/integration/app/test_app.py | 184 ++ tests/unit/app/__init__.py | 0 tests/unit/app/test_api.py | 476 +++++ tests/unit/app/test_auth.py | 72 + tests/unit/app/test_deploy.py | 253 +++ uv.lock | 537 ++++- 56 files changed, 5701 insertions(+), 9 deletions(-) create mode 100644 app-ui/.gitignore create mode 100644 app-ui/app/assessment/page.tsx create mode 100644 app-ui/app/globals.css create mode 100644 app-ui/app/layout.tsx create mode 100644 app-ui/app/mapping/page.tsx create mode 100644 app-ui/app/migration/page.tsx create mode 100644 app-ui/app/page.tsx create mode 100644 app-ui/components/runs.tsx create mode 100644 app-ui/components/shell.tsx create mode 100644 app-ui/components/ui/alert.tsx create mode 100644 app-ui/components/ui/badge.tsx create mode 100644 app-ui/components/ui/button.tsx create mode 100644 app-ui/components/ui/card.tsx create mode 100644 app-ui/components/ui/checkbox.tsx create mode 100644 app-ui/components/ui/confirm-dialog.tsx create mode 100644 app-ui/components/ui/input.tsx create mode 100644 app-ui/next.config.ts create mode 100644 app-ui/package.json create mode 100644 app-ui/pnpm-lock.yaml create mode 100644 app-ui/postcss.config.mjs create mode 100644 app-ui/tsconfig.json create mode 100644 docs/ucx/docs/reference/app.mdx create mode 100644 src/databricks/labs/ucx/app/__init__.py create mode 100644 src/databricks/labs/ucx/app/auth.py create mode 100644 src/databricks/labs/ucx/app/bundle/app.yaml create mode 100644 src/databricks/labs/ucx/app/bundle/databricks.yml create mode 100644 src/databricks/labs/ucx/app/context.py create mode 100644 src/databricks/labs/ucx/app/deploy.py create mode 100644 src/databricks/labs/ucx/app/main.py create mode 100644 src/databricks/labs/ucx/app/models.py create mode 100644 src/databricks/labs/ucx/app/routes/__init__.py create mode 100644 src/databricks/labs/ucx/app/routes/assessment.py create mode 100644 src/databricks/labs/ucx/app/routes/dependencies.py create mode 100644 src/databricks/labs/ucx/app/routes/mapping.py create mode 100644 src/databricks/labs/ucx/app/routes/migration.py create mode 100644 src/databricks/labs/ucx/app/routes/runs.py create mode 100644 src/databricks/labs/ucx/app/routes/status.py create mode 100644 src/databricks/labs/ucx/app/service.py create mode 100644 tests/integration/app/__init__.py create mode 100644 tests/integration/app/test_app.py create mode 100644 tests/unit/app/__init__.py create mode 100644 tests/unit/app/test_api.py create mode 100644 tests/unit/app/test_auth.py create mode 100644 tests/unit/app/test_deploy.py diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 1c90db4a81..cbb9711967 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -123,3 +123,57 @@ jobs: run: | # Exit with status code 1 if there are differences (i.e. unformatted files) git diff --exit-code + + app: + # The UCX Databricks App: type-check and build the frontend, then build the wheel and check that it contains the + # app. Release builds must run `make build` the same way, see docs/ucx/docs/dev/contributing.mdx. + runs-on: + group: databrickslabs-protected-runner-group + labels: linux-ubuntu-latest + permissions: + # Authenticate with JFrog for package resolution. + id-token: write + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup uv + uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 + with: + version: "0.13.7" + checksum: "c5221ff41b7892bfc223543883600125f6a47fe845093e0f5bdff4f359196d83" + + - name: Setup Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "26" + + - name: Setup pnpm + uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1 + with: + version: "9.15.9" + + # After the tools: it configures npm (and so pnpm) for JFrog, with ignore-scripts=true. + - name: Setup for JFrog + uses: ./.github/actions/jfrog-auth + + - name: Type-check the frontend + working-directory: app-ui + run: | + pnpm install --frozen-lockfile + pnpm typecheck + + - name: Build the wheel with the app + run: make build + + - name: Check that the wheel contains the app + run: | + set -euo pipefail + wheel="$(ls dist/databricks_labs_ucx-*-py3-none-any.whl)" + for required in databricks/labs/ucx/app/static/index.html databricks/labs/ucx/app/bundle/requirements.txt \ + databricks/labs/ucx/app/bundle/databricks.yml databricks/labs/ucx/app/bundle/app.yaml; do + if ! unzip -l "$wheel" "$required" > /dev/null; then + printf '::error::%s\n' "$wheel is missing $required" + exit 1 + fi + done diff --git a/.gitignore b/.gitignore index ff48d3a0cd..05d709bba5 100644 --- a/.gitignore +++ b/.gitignore @@ -153,3 +153,7 @@ dev/cleanup.py .databricks-login.json *.out foo + +# UCX app frontend, built by `make app-ui` +src/databricks/labs/ucx/app/static/ +src/databricks/labs/ucx/app/bundle/requirements.txt diff --git a/Makefile b/Makefile index 7b2ece59c7..d345ddf915 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,22 @@ coverage: $(UV_TEST) --cov src tests/unit --cov-report=html --cov-fail-under=89 open htmlcov/index.html -build: +# The app frontend is a static export, built into the Python package so that it ships in the wheel. +APP_UI_STATIC := src/databricks/labs/ucx/app/static + +app-ui: + cd app-ui && pnpm install --frozen-lockfile && NEXT_TELEMETRY_DISABLED=1 pnpm build + rm -rf $(APP_UI_STATIC) + cp -R app-ui/out $(APP_UI_STATIC) + +# Hashed requirements of the app, from uv.lock: `deploy-app` installs exactly these in the app environment. +APP_REQUIREMENTS := src/databricks/labs/ucx/app/bundle/requirements.txt + +app-requirements: + uv export --frozen --extra app --no-default-groups --no-emit-project --no-header --no-annotate \ + --format requirements-txt --output-file $(APP_REQUIREMENTS) + +build: app-ui app-requirements uv build --require-hashes --build-constraints=.build-constraints.txt lock-dependencies: UV_FROZEN := 0 @@ -74,4 +89,4 @@ docs-serve: docs-build yarn --cwd docs/ucx serve .DEFAULT: all -.PHONY: all clean dev lint fmt test integration coverage build lock-dependencies known cmds solacc docs-install docs-serve-dev docs-build docs-serve +.PHONY: all clean dev lint fmt test integration coverage app-ui app-requirements build lock-dependencies known cmds solacc docs-install docs-serve-dev docs-build docs-serve diff --git a/README.md b/README.md index eb1df26f4a..3bb453bf7d 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,9 @@ UCX is a toolkit for enabling Unity Catalog (UC) in your Databricks workspace. U Please refer to the [UCX documentation](https://databrickslabs.github.io/ucx/) for detailed information on how to use UCX. +To run the assessment, review the table mapping and migrate tables from a web UI, deploy the +[UCX App](https://databrickslabs.github.io/ucx/docs/reference/app) with `databricks labs ucx deploy-app`. + # Developer documentation diff --git a/app-ui/.gitignore b/app-ui/.gitignore new file mode 100644 index 0000000000..29c353c977 --- /dev/null +++ b/app-ui/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +.next/ +out/ +next-env.d.ts +*.tsbuildinfo diff --git a/app-ui/app/assessment/page.tsx b/app-ui/app/assessment/page.tsx new file mode 100644 index 0000000000..a24a053082 --- /dev/null +++ b/app-ui/app/assessment/page.tsx @@ -0,0 +1,93 @@ +"use client"; + +import { ExternalLink, Play } from "lucide-react"; +import { useState } from "react"; +import useSWR from "swr"; + +import { useRuns } from "@/components/runs"; +import { PageTitle } from "@/components/shell"; +import { Alert } from "@/components/ui/alert"; +import { Button } from "@/components/ui/button"; +import { Card, CardTitle } from "@/components/ui/card"; +import { Checkbox } from "@/components/ui/checkbox"; +import { api, fetcher } from "@/lib/api"; +import type { Dashboard } from "@/lib/types"; + +export default function AssessmentPage() { + const { track } = useRuns(); + const [forceRefresh, setForceRefresh] = useState(false); + const [starting, setStarting] = useState(false); + const [runError, setRunError] = useState(null); + const { data: dashboard, error: dashboardError } = useSWR("/api/assessment/dashboard", fetcher); + + async function runAssessment() { + setStarting(true); + setRunError(null); + try { + const run = await api.runAssessment(forceRefresh); + track({ run_id: run.run_id, run_url: run.run_url, name: run.workflow }); + } catch (e) { + setRunError((e as Error).message); + } finally { + setStarting(false); + } + } + + return ( +
+ + + Run assessment +
+ + + +
+ {runError ? ( + {runError} + ) : ( + + The run starts in the background. Progress shows in the Active runs window. + + )} +
+ + + + Open in new tab + + + ) + } + > + Assessment dashboard + + {dashboardError && {dashboardError.message}} + {dashboard && ( + <> +