diff --git a/.gitignore b/.gitignore index c53da76a2..b8ecbe8ef 100644 --- a/.gitignore +++ b/.gitignore @@ -193,3 +193,9 @@ test.py sqlbot-xpack .claude +AGENTS.local.env + +# 本机环境配置与运行日志 +frontend/.npmrc +backend/logs/ +/logs/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..8624edc68 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,50 @@ +# SQLBot Agent 指南 + +SQLBot 让业务用户用自然语言提问,基于已配置的数据源生成并安全执行 SQL,返回数据、图表、分析和后续问题建议。领域词汇表见 `CONTEXT.md`。 + +## 仓库结构 + +| 路径 | 职责 | +| --- | --- | +| `backend/` | Python 3.11 / FastAPI / SQLModel 后端。`main.py` 组装应用、MCP、中间件、静态资源和 xpack;业务域在 `apps/`,共享代码在 `common/`,迁移在 `alembic/versions/`,全部测试(含仓库守卫)在 `tests/`。 | +| `frontend/` | Vue 3、TypeScript strict、Vite、Pinia、Vue Router 和 Element Plus。请求在 `src/api/`,共享实体在 `src/entity/`,状态在 `src/stores/`,路由在 `src/router/`,UI 在 `src/views/` 和 `src/components/`。 | +| `g2-ssr/` | Node.js 图表渲染服务,使用 `@antv/g2-ssr`;图表实现位于 `charts/`,目录内有独立 `AGENTS.md`。 | +| `installer/` | 离线安装、卸载、配置模板和服务控制脚本。 | +| `Dockerfile*`、`docker-compose.yaml`、`start.sh` | 容器组装和运行时进程启动。 | + +商业扩展源码维护在独立仓库 [dataease/sqlbot-xpack](https://github.com/dataease/sqlbot-xpack)。它不是 submodule,也不是本仓库的固定子目录。本地 `sqlbot-xpack/` checkout 会被主仓库忽略,永远不应出现在本仓库 PR 中。 + +## xpack 工作流 + +默认使用开源版 SQLBot 工作流:把 `sqlbot-xpack` 视为版本范围由 `backend/pyproject.toml` 约束的已发布 wheel(精确版本冻结在不入库的 `uv.lock`)。不读取、不修改本地 xpack checkout。 + +只有任务确实需要修改或调试闭源 xpack 代码,或需要两个仓库联动验证时,才按 `docs/agents/xpack.md` 检查本地关联开关(`AGENTS.local.env`)并协调两个仓库的变更。 + +## 按需文档 + +| 触发条件 | 必读文档 | +| --- | --- | +| 修改业务逻辑、数据模型(SQLModel)、权限、Chat 问题流程、助手集成、前端信息架构,或需要命名和领域术语 | `CONTEXT.md` | +| 修改后端业务代码 | `docs/agents/backend.md` | +| 修改前端代码 | `docs/agents/frontend.md` | +| 修改或新增测试、执行验证 | `docs/agents/testing.md` | +| 修改用户可见文案或新增语言 | `docs/agents/i18n.md` | +| 修改认证、授权、SQL 执行、上传/下载、嵌入协议或前端渲染安全 | `docs/agents/security.md` | +| 修改 SQLModel 模型或 Alembic 迁移 | `docs/agents/migrations.md` | +| 修改 Dockerfile、installer、GitHub Actions 或发布产物 | `docs/agents/packaging.md` | +| 修改或调试闭源 xpack 代码、双仓库联动验证 | `docs/agents/xpack.md` | +| 修改图表渲染服务、后端图表配置或图表字段/输出契约 | `g2-ssr/AGENTS.md` | +| 领域边界仍不明确 | `docs/agents/domain-open-questions.md`,并向使用者确认 | + +## 全局硬规则 + +- 在正确仓库检查 status/diff;SQLBot 主仓库和 xpack 独立仓库不要混出同一个提交。 +- 不要提交日志、构建产物、`.env` 值、密钥、本地路径、私有 registry 配置或生成的 xpack 产物。 +- 提交信息和 PR 描述不添加 `Co-Authored-By`、"Generated with" 等任何 AI 工具署名行。 +- 提交信息沿用仓库既有 conventional 风格:`fix:`、`feat:`、`refactor:` 等前缀(可带 scope),单行概述。 +- 不要为了通过测试削弱安全守卫;安全、权限、SQL、Host、路径和嵌入认证改动必须有相关回归验证。 +- 修改 Docker、installer 或路径配置时,核对前端构建产物、后端工作目录、`/opt/sqlbot` 数据目录、图表输出和日志挂载仍然一致。 +- 依赖、lockfile 和版本号只在任务明确需要时更新;不要顺手刷新。 +- 验证以构建和测试为准;除非用户明确要求,不构建 Docker 镜像、不启动完整运行栈。 +- 除非用户明确要求,不要上传、发布或推送镜像 / wheel / 包。 +- 变更涉及本文件或 `docs/agents/` 描述的约定(目录职责、命令、流程)时,同步更新对应文档。 diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 000000000..905a2a29b --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,83 @@ +# SQLBot + +SQLBot 让业务用户用自然语言提问,基于已配置的数据源生成并安全执行 SQL,然后返回数据、图表、分析和后续问题建议。 + +## Language + +### 工作空间与数据 + +**Workspace / 工作空间**: +用户、会话、数据源、模型等 SQLBot 资源的隔离边界。 +_Avoid_: Organization、tenant + +**Workspace ID / 工作空间 ID**: +工作空间的标识。历史上的 `oid` 和 `workspace_id` 字段表示同一个概念。 +_Avoid_: 把 `oid` 理解成独立的组织概念 + +**Datasource / 数据源**: +已配置的外部数据源,以及 SQLBot 用于生成查询的表、字段、关系和 embedding 等元数据。 +_Avoid_: Database、connection + +**SQL Example / SQL 示例**: +用于引导 SQL 生成的“问题 + SQL”示例。 +_Avoid_: Data training、training data + +**Terminology / 术语**: +业务词或短语的解释,可包含同义词,用于提升问题和表结构理解。 +_Avoid_: Custom prompt、SQL example + +**Custom Prompt / 自定义提示词**: +附加在模型任务上的场景指令,可按工作空间、数据源或助手场景生效。 +_Avoid_: Terminology、SQL example + +### 会话 + +**Chat / 会话**: +用户在一个工作空间内连续提出数据问题的对话。 +_Avoid_: Assistant、dashboard + +**Chat Record / 会话记录**: +一次问题执行的可持久化结果,包含问题、生成 SQL、查询结果、图表配置、错误以及关联的后续记录。 +_Avoid_: Chat + +**Analysis / 分析**: +基于既有问题结果的模型生成解读。 +_Avoid_: Prediction + +**Prediction / 预测**: +基于既有问题结果的模型生成前瞻性估计。 +_Avoid_: Analysis + +**Recommended Problem / 推荐问题**: +与数据源关联的已配置问题。 +_Avoid_: Guess question + +**Guess Question / 猜测问题**: +由模型根据会话上下文推测的后续问题。 +_Avoid_: Recommended problem + +### 助手与集成 + +**Assistant / 助手**: +把 SQLBot 问数能力暴露给外部系统的集成配置。 +_Avoid_: Chat + +**Ordinary Assistant / 普通小助手**: +标准的小助手集成形态。 +_Avoid_: Advanced assistant、page-embedded assistant + +**Advanced Assistant / 高级应用**: +面向更深集成场景的高级小助手形态。 +_Avoid_: Ordinary assistant、page-embedded assistant + +**Page-embedded Assistant / 页面嵌入助手**: +用于把 SQLBot 页面嵌入目标系统的小助手形态。 +_Avoid_: Ordinary assistant、advanced assistant + +**Assistant Domain / 助手目标域名**: +小助手对接的外部目标系统域名。 +_Avoid_: Business domain、workspace + +**Dashboard / 仪表板**: +为重复分析保存的数据视图集合。 +_Avoid_: Chat、chat record diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 9c9f93d08..57533423f 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -127,7 +127,7 @@ strict = true exclude = ["venv", ".venv", "alembic"] [tool.ruff] -target-version = "py310" +target-version = "py311" exclude = ["alembic"] [tool.ruff.lint] diff --git a/backend/scripts/lint.sh b/backend/scripts/lint.sh deleted file mode 100644 index b3b2b4ecc..000000000 --- a/backend/scripts/lint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -mypy app -ruff check app -ruff format app --check diff --git a/backend/scripts/prestart.sh b/backend/scripts/prestart.sh deleted file mode 100755 index 1b395d513..000000000 --- a/backend/scripts/prestart.sh +++ /dev/null @@ -1,13 +0,0 @@ -#! /usr/bin/env bash - -set -e -set -x - -# Let the DB start -python app/backend_pre_start.py - -# Run migrations -alembic upgrade head - -# Create initial data in DB -python app/initial_data.py diff --git a/backend/scripts/test.sh b/backend/scripts/test.sh deleted file mode 100755 index df23f702e..000000000 --- a/backend/scripts/test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -coverage run --source=app -m pytest -coverage report --show-missing -coverage html --title "${@-coverage}" diff --git a/backend/scripts/tests-start.sh b/backend/scripts/tests-start.sh deleted file mode 100755 index 89dcb0da2..000000000 --- a/backend/scripts/tests-start.sh +++ /dev/null @@ -1,7 +0,0 @@ -#! /usr/bin/env bash -set -e -set -x - -python app/tests_pre_start.py - -bash scripts/test.sh "$@" diff --git a/tests/test_cwe89_escape_fix.py b/backend/tests/test_cwe89_escape_fix.py similarity index 97% rename from tests/test_cwe89_escape_fix.py rename to backend/tests/test_cwe89_escape_fix.py index 343ebde09..241a5fa89 100644 --- a/tests/test_cwe89_escape_fix.py +++ b/backend/tests/test_cwe89_escape_fix.py @@ -6,17 +6,21 @@ 2. _escape_sql_value() preserves safe values unchanged 3. _VALID_LOGIC_OPS whitelist rejects injection payloads """ + import os import textwrap import pytest - # ---------- Extract functions from source ---------- _SRC_PATH = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "backend", "apps", "datasource", "crud", "row_permission.py", + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), + "backend", + "apps", + "datasource", + "crud", + "row_permission.py", ) # Parse the source and extract _escape_sql_value function body @@ -55,6 +59,7 @@ def _escape_sql_value(value): # Test _escape_sql_value # ============================================================ + class TestEscapeSqlValue: """Tests for the _escape_sql_value helper.""" @@ -142,6 +147,7 @@ def test_backslash_quote_bypass_attempt(self): # Test _VALID_LOGIC_OPS whitelist # ============================================================ + class TestValidLogicOps: """Tests for the logic operator whitelist.""" @@ -181,6 +187,7 @@ def test_case_insensitive_validation(self): # Test SQL fragment construction safety # ============================================================ + class TestSqlFragmentSafety: """End-to-end tests simulating how escaped values are used in SQL fragments.""" diff --git a/tests/test_distributed_lock.py b/backend/tests/test_distributed_lock.py similarity index 98% rename from tests/test_distributed_lock.py rename to backend/tests/test_distributed_lock.py index 475fc2952..dea383035 100644 --- a/tests/test_distributed_lock.py +++ b/backend/tests/test_distributed_lock.py @@ -8,8 +8,7 @@ from sqlalchemy.engine import Connection from sqlalchemy.exc import SQLAlchemyError - -BACKEND_DIR = Path(__file__).resolve().parents[1] / "backend" +BACKEND_DIR = Path(__file__).resolve().parents[1] sys.path.insert(0, str(BACKEND_DIR)) from common.utils import distributed_lock as lock_module # noqa: E402 diff --git a/tests/test_embedded_auth_bypass_fix.py b/backend/tests/test_embedded_auth_bypass_fix.py similarity index 74% rename from tests/test_embedded_auth_bypass_fix.py rename to backend/tests/test_embedded_auth_bypass_fix.py index e561601ce..689949202 100644 --- a/tests/test_embedded_auth_bypass_fix.py +++ b/backend/tests/test_embedded_auth_bypass_fix.py @@ -11,18 +11,20 @@ 3. Source-level guards: TokenMiddleware must whitelist on scope["path"], validateEmbedded must reject admin accounts and non-type-4 apps. """ + import os import re import textwrap import pytest - # ---------- Paths to sources ---------- -_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -_HOST_VALIDATION_SRC = os.path.join(_ROOT, "backend", "common", "core", "host_validation.py") +_HOST_VALIDATION_SRC = os.path.join( + _ROOT, "backend", "common", "core", "host_validation.py" +) _WHITELIST_SRC = os.path.join(_ROOT, "backend", "common", "utils", "whitelist.py") _AUTH_SRC = os.path.join(_ROOT, "backend", "apps", "system", "middleware", "auth.py") @@ -53,35 +55,42 @@ # Test Host header validation # ============================================================ + class TestHostValidation: """Valid Host headers pass; path-carrying / malformed ones are rejected.""" - @pytest.mark.parametrize("host", [ - "localhost", - "localhost:8000", - "127.0.0.1", - "127.0.0.1:8000", - "example.com", - "api.example.com:443", - "[::1]", - "[::1]:8000", - "10.0.0.1", - "a.b.c.d.e.f.g", - ]) + @pytest.mark.parametrize( + "host", + [ + "localhost", + "localhost:8000", + "127.0.0.1", + "127.0.0.1:8000", + "example.com", + "api.example.com:443", + "[::1]", + "[::1]:8000", + "10.0.0.1", + "a.b.c.d.e.f.g", + ], + ) def test_valid_host_accepted(self, host): assert _HOST_RE.match(host) is not None - @pytest.mark.parametrize("host", [ - "", # empty - "evil.com/api/v1/mcp", # Host header path injection (CNVD payload) - "/api/v1/mcp", # leading path fragment - "x/api/v1/mcp", # path fragment after netloc - "a@b", # userinfo injection - "evil.com/path?x=1", # query fragment - "evil.com#frag", # fragment - "evil com", # whitespace - "evil.com\nX-Real-IP: 1.2.3.4", # header injection attempt - ]) + @pytest.mark.parametrize( + "host", + [ + "", # empty + "evil.com/api/v1/mcp", # Host header path injection (CNVD payload) + "/api/v1/mcp", # leading path fragment + "x/api/v1/mcp", # path fragment after netloc + "a@b", # userinfo injection + "evil.com/path?x=1", # query fragment + "evil.com#frag", # fragment + "evil com", # whitespace + "evil.com\nX-Real-IP: 1.2.3.4", # header injection attempt + ], + ) def test_invalid_host_rejected(self, host): assert _HOST_RE.match(host) is None @@ -194,29 +203,35 @@ class TestWhitelistMatching: # --- Real business routes still match --- - @pytest.mark.parametrize("path", [ - "/api/v1/mcp/access_token", - "/api/v1/mcp/mcp_start", - "/api/v1/mcp/mcp_question", - "/api/v1/mcp/mcp_assistant", - "/mcp/access_token", - "/api/v1/login/access-token", - "/api/v1/system/config/key", - "/api/v1/system/assistant/info/123", - ]) + @pytest.mark.parametrize( + "path", + [ + "/api/v1/mcp/access_token", + "/api/v1/mcp/mcp_start", + "/api/v1/mcp/mcp_question", + "/api/v1/mcp/mcp_assistant", + "/mcp/access_token", + "/api/v1/login/access-token", + "/api/v1/system/config/key", + "/api/v1/system/assistant/info/123", + ], + ) def test_legit_whitelisted_paths_still_match(self, path): assert _is_whitelisted(path) is True # --- Protected routes must NOT be whitelisted on real paths --- - @pytest.mark.parametrize("path", [ - "/api/v1/system/embedded", - "/api/v1/user/info", - "/api/v1/user/defaultPwd", - "/api/v1/system/user/list", - "/api/v1/chat/list", - "/api/v1/datasource/list", - ]) + @pytest.mark.parametrize( + "path", + [ + "/api/v1/system/embedded", + "/api/v1/user/info", + "/api/v1/user/defaultPwd", + "/api/v1/system/user/list", + "/api/v1/chat/list", + "/api/v1/datasource/list", + ], + ) def test_protected_paths_not_whitelisted(self, path): assert _is_whitelisted(path) is False @@ -235,25 +250,31 @@ def test_injected_path_documented_as_defense_in_depth(self): # Source-level regression guards # ============================================================ + class TestSourceLevelGuards: """Pin the actual fix points in source to prevent regressions.""" def test_auth_middleware_uses_scope_path(self): - assert "request.scope.get(\"path\")" in _auth_source, \ + assert 'request.scope.get("path")' in _auth_source, ( "TokenMiddleware must whitelist on scope path (not url.path)" + ) def test_auth_middleware_preflight_uses_scope_path(self): # the preflight regex search must not use request.url.path - assert "re.search(r'/system/assistant/info/(\\d+)', request_path)" in _auth_source + assert ( + "re.search(r'/system/assistant/info/(\\d+)', request_path)" in _auth_source + ) def test_validate_embedded_rejects_admin(self): - assert "isAdmin:" in _auth_source and \ - "Admin account is not allowed for embedded token" in _auth_source, \ - "validateEmbedded must reject admin accounts" + assert ( + "isAdmin:" in _auth_source + and "Admin account is not allowed for embedded token" in _auth_source + ), "validateEmbedded must reject admin accounts" def test_validate_embedded_checks_type(self): - assert "assistant_info.type != 4" in _auth_source, \ + assert "assistant_info.type != 4" in _auth_source, ( "validateEmbedded must only accept type=4 embedded apps" + ) def test_host_validation_middleware_exists(self): assert "class HostValidationMiddleware" in _host_validation_source @@ -262,8 +283,9 @@ def test_host_validation_registered(self): main_src_path = os.path.join(_ROOT, "backend", "main.py") with open(main_src_path) as f: main_source = f.read() - assert "app.add_middleware(HostValidationMiddleware)" in main_source, \ + assert "app.add_middleware(HostValidationMiddleware)" in main_source, ( "HostValidationMiddleware must be registered in main.py" + ) if __name__ == "__main__": diff --git a/tests/test_execution_error_details.py b/backend/tests/test_execution_error_details.py similarity index 88% rename from tests/test_execution_error_details.py rename to backend/tests/test_execution_error_details.py index 79729aab2..b9f6659ed 100644 --- a/tests/test_execution_error_details.py +++ b/backend/tests/test_execution_error_details.py @@ -4,7 +4,7 @@ import unittest from pathlib import Path -PROJECT_ROOT = Path(__file__).resolve().parents[1] +PROJECT_ROOT = Path(__file__).resolve().parents[2] CHAT_DIR = PROJECT_ROOT / "frontend" / "src" / "views" / "chat" COMPONENT_DIR = CHAT_DIR / "execution-component" @@ -74,17 +74,17 @@ def test_log_components_render_the_forwarded_error(self) -> None: self.assertIsNotNone(error_branch) self.assertIn("{{ error }}", error_branch.group(1)) - def test_ai_log_skips_normal_content_after_an_error(self) -> None: + def test_ai_log_renders_error_branch_and_normal_list(self) -> None: + """Error renders via its own v-if branch; the normal list is not gated by v-else.""" source = read_source(COMPONENT_DIR / "LogWithAi.vue") self.assertRegex( source, re.compile( - r'.*?\s*' - r'