Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .changes/hister-content.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
增加普通 Markdown 源文解释与文字能力,并保持 HTML、纯文等既有内容合同。
2 changes: 1 addition & 1 deletion app/business/info_base/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@
## Resolver Boundary

- `ResolverManager` 只按 exact ID select/register;unknown ID 明确失败,不存在 default decoder fallback。
- Core bootstrap 独立于 extension loading,注册九个 `core.<kind>.v1` semantic resolver。
- Core bootstrap 独立于 extension loading,注册十个 `core.<kind>.v1` semantic resolver。
- Duplicate registration:同 class 重复注册 idempotent,不同 class 抢同 ID 抛错。
- `get_text()` 的 unsupported、supported-null 与 authored-empty 必须保持可区分;不要添加 use-specific
`get_*_for_embedding()` projection。
Expand Down
8 changes: 4 additions & 4 deletions app/business/info_base/resolver/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@
## 关键文件

- `main.py`:exact registry、resolver instance、relation/solved caches;
- `contracts.py`:九个 core IDs 与 typed failures;
- `contracts.py`:十个 core IDs 与 typed failures;
- `bootstrap.py`:core decoder registration;
- `inspection.py`:bounded byte inspection helpers;
- `label.py`:shared whitespace normalization and 96-code-point identifier bound;
- `{text,html,image,audio,video,pdf,epub,zip,file}.py`:exact core contract versions。
- `{text,html,markdown,image,audio,video,pdf,epub,zip,file}.py`:exact core contract versions。

## Construction And Registration

Expand All @@ -30,7 +30,7 @@
使 persisted Block decoder 在 active Extension route/resource 关闭后仍可用;exact version replacement 仍以进程
restart 为边界。
- `register_core_resolvers()` 独立于 extension sync/start;extension disabled 不能让 core decoder 消失。
- Resolver ID 必须 namespaced + versioned。共享 semantic IDs 固定为九个 `core.<kind>.v1`;不保留 bare alias。
- Resolver ID 必须 namespaced + versioned。共享 semantic IDs 固定为十个 `core.<kind>.v1`;不保留 bare alias。
- Agent graph drafting 是 opt-in capability:subclass 同时声明 `draft_description`、`draft_input_model` 并实现
`create_graph(input) -> StarsGraphForm`。普通 decode-capable Resolver 不会因此自动成为 draft-capable。

Expand All @@ -53,7 +53,7 @@ Concrete resolver 必须实现:
`UnsupportedResolverCapability` 表示 exact contract 没有该能力;`None` 表示能力存在但当前 block 没有有意义的
result;empty string 只能来自真实 authored/derived empty。`UnknownResolverError` 是另一类 registry failure。

九个 core semantic resolvers 中,text/HTML 提供 text;image/audio/video/PDF/EPUB/ZIP/file 在没有真实
十个 core semantic resolvers 中,text/HTML/Markdown 提供 text;image/audio/video/PDF/EPUB/ZIP/file 在没有真实
OCR/STT/extraction 能力前显式 unsupported,不用 metadata 伪装正文。Embedding/retrieval owner 只消费这个
通用 projection,不要求 resolver 理解 AI model/profile。

Expand Down
2 changes: 2 additions & 0 deletions app/business/info_base/resolver/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
"PDFResolver",
"VideoResolver",
"TextResolver",
"MarkdownResolver",
"HTMLResolver",
"ZIPResolver",
]
Expand All @@ -54,5 +55,6 @@
from .pdf import PDFResolver
from .video import VideoResolver
from .text import TextResolver
from .markdown import MarkdownResolver
from .html import HTMLResolver
from .zip import ZIPResolver
2 changes: 2 additions & 0 deletions app/business/info_base/resolver/bootstrap.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ def register_core_resolvers() -> None:
from .image import ImageResolver
from .pdf import PDFResolver
from .text import TextResolver
from .markdown import MarkdownResolver
from .video import VideoResolver
from .zip import ZIPResolver

for resolver_class in (
TextResolver,
MarkdownResolver,
HTMLResolver,
ImageResolver,
AudioResolver,
Expand Down
2 changes: 2 additions & 0 deletions app/business/info_base/resolver/contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

CoreResolverID: TypeAlias = Literal[
"core.text.v1",
"core.markdown.v1",
"core.html.v1",
"core.image.v1",
"core.audio.v1",
Expand All @@ -22,6 +23,7 @@

CORE_RESOLVER_IDS: tuple[CoreResolverID, ...] = (
"core.text.v1",
"core.markdown.v1",
"core.html.v1",
"core.image.v1",
"core.audio.v1",
Expand Down
9 changes: 9 additions & 0 deletions app/business/info_base/resolver/html.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,15 @@ async def get_text(
refresh=refresh,
materialize_missing=materialize_missing,
)
if any(f"<{tag}" in source.lower() for tag in ("video", "audio", "track")):
document = lxml_html.document_fromstring(source)
for resource in document.xpath("//video | //audio | //source | //track"):
for attribute in ("src", "poster"):
if address := resource.get(attribute):
link = lxml_html.Element("a", href=address)
link.text = resource.get("label") or address
resource.addnext(link)
source = lxml_html.tostring(document, encoding="unicode")
return await asyncio.to_thread(html2text.HTML2Text().handle, source)

async def get_label(self, *, refresh: bool = False) -> str:
Expand Down
1 change: 1 addition & 0 deletions app/business/info_base/resolver/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@ def match_media_type(cls, media_type: str | None) -> ResolverType | None:

exact = {
"text/plain": "core.text.v1",
"text/markdown": "core.markdown.v1",
"text/html": "core.html.v1",
"application/xhtml+xml": "core.html.v1",
"application/pdf": "core.pdf.v1",
Expand Down
35 changes: 35 additions & 0 deletions app/business/info_base/resolver/markdown.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
"""Markdown source remains the exact text projection; rendering is a reader concern."""

from .inspection import decode_unicode_bytes
from .label import format_label
from .main import Resolver, TextProjectionContext


class MarkdownResolver(Resolver[str, str | bytes], rso_type="core.markdown.v1"):
async def _get_solved_content(
self, *, refresh: bool = False, materialize_missing: bool = True
) -> str:
content = await self.get_raw_content(refresh=refresh)
return (
content
if isinstance(content, str)
else decode_unicode_bytes(content, self.__rsotype__)
)

async def get_text(
self,
*,
context: TextProjectionContext = "default",
refresh: bool = False,
materialize_missing: bool = True,
) -> str:
return await self.get_solved_content(
refresh=refresh, materialize_missing=materialize_missing
)

async def get_label(self, *, refresh: bool = False) -> str:
return format_label(
"markdown",
await self.get_text(refresh=refresh, materialize_missing=False),
first_line=True,
)
2 changes: 1 addition & 1 deletion app/version.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from pathlib import Path
import tomllib

CORE_VERSION = "0.3.3"
CORE_VERSION = "0.3.4"
APPLICATION_VERSION = tomllib.loads(
(Path(__file__).parents[1] / "pyproject.toml").read_text(encoding="utf-8")
)["project"]["version"]
4 changes: 2 additions & 2 deletions docs/30-unit-tdd/local-files-extension.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# 本地文件读取分发

`extensions/local_files` 交付 `inkcre/local-files` 的 Python 读取能力,解释 `extensions.local_files.file.v1` 与 `body` 关系。源合同在 Hub 的 `20-product-tdd/local-files-contract.md`。目录监视、文件提取及共享写入由具备文件权限的执行分发完成;Python 分发不提供收入 API,也不注册目录 Source 占位类。
`extensions/local_files` 交付 `inkcre/local-files` 的 Python 读取能力,解释 `extensions.local_files.file.v1`/`file.v2` 与 `body` 关系。源合同在 Hub 的 `20-product-tdd/local-files-contract.md`。目录监视、文件提取及共享写入由具备文件权限的执行分发完成;Python 分发不提供收入 API,也不注册目录 Source 占位类。

`LocalFileResolver` 读取元数据并校验唯一的 `core.text.v1` 正文,允许首次失败时尚无正文。原件失效或当前提取失败仍返回最后成功正文,同时保留状态和提取范围。位置只作为文字解释,不访问本机路径,不尝试远端取文件,也不物化其他副本。此功能复用已有 Source/Job/图模型,没有新增迁移。
`LocalFileResolver` 保持 v1 的唯一 inline `core.text.v1` 正文合同;`LocalFileV2Resolver` 按正文实际采用的 text、HTML 或 Markdown resolver 生成文字投影,允许首次失败时尚无正文。原件失效或当前提取失败仍返回最后成功正文,同时保留状态和提取范围。位置只作为文字解释,不访问本机路径,不尝试远端取文件,也不物化其他副本。此功能复用已有 Source/Job/图模型,没有新增迁移。
1 change: 1 addition & 0 deletions extensions/local_files/.changes/hister-content.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
读取 Local Files v2 的 HTML、Markdown 正文,同时保持 v1 文字合同和失败保留。
2 changes: 1 addition & 1 deletion extensions/local_files/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ local_files = "extensions.local_files:Extension"
name = "inkcre/local-files"
nickname = "本地文件"
host-sdk = "core-py"
host-sdk-version = ">=0.3.3 <0.4.0"
host-sdk-version = ">=0.3.4 <0.4.0"

[tool.setuptools]
packages = ["extensions.local_files"]
Expand Down
33 changes: 29 additions & 4 deletions extensions/local_files/resolver.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
"""Read persisted text without accessing the original Peer or executing a Source."""

from pathlib import PurePosixPath
from typing import ClassVar

from app.business.info_base.resolver import Resolver, TextProjectionContext
from app.business.info_base.resolver import Resolver, ResolverManager, TextProjectionContext
from app.business.info_base.resolver.label import format_label
from app.business.info_base.services import BlockService

from .schema import FILE_RESOLVER, FileMetadata, SolvedFile, file_location
from .schema import FILE_RESOLVER, FILE_RESOLVER_V2, FileMetadata, SolvedFile, file_location


class LocalFileResolver(Resolver[SolvedFile, str], rso_type=FILE_RESOLVER):
body_resolvers: ClassVar[tuple[str, ...]] = ("core.text.v1",)

async def _get_solved_content(
self, *, refresh: bool = False, materialize_missing: bool = True
) -> SolvedFile:
Expand All @@ -23,9 +26,23 @@ async def _get_solved_content(
if not bodies:
return SolvedFile(metadata=metadata, body_id=None, text=None)
block = await BlockService.get(bodies[0].to_)
if block is None or block.resolver != "core.text.v1" or block.storage is not None:
if (
block is None
or block.resolver not in self.body_resolvers
or block.storage is not None
):
raise ValueError("文件正文不存在或类型不符")
return SolvedFile(metadata=metadata, body_id=block.id, text=block.content)
return SolvedFile(
metadata=metadata,
body_id=block.id,
text=(
block.content
if block.resolver == "core.text.v1"
else await ResolverManager.get(block).get_text(
refresh=refresh, materialize_missing=materialize_missing
)
),
)

async def get_text(
self,
Expand All @@ -51,3 +68,11 @@ async def get_label(self, *, refresh: bool = False) -> str:
metadata = FileMetadata.model_validate_json(await self.get_raw_content(refresh=refresh))
_, path = file_location(metadata.location)
return format_label("本地文件", PurePosixPath(path).name)


class LocalFileV2Resolver(LocalFileResolver, rso_type=FILE_RESOLVER_V2):
body_resolvers: ClassVar[tuple[str, ...]] = (
"core.text.v1",
"core.html.v1",
"core.markdown.v1",
)
1 change: 1 addition & 0 deletions extensions/local_files/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, field_validator

FILE_RESOLVER = "extensions.local_files.file.v1"
FILE_RESOLVER_V2 = "extensions.local_files.file.v2"


def file_location(value: str) -> tuple[UUID, str]:
Expand Down
46 changes: 46 additions & 0 deletions tests/extensions/local_files/test_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,3 +64,49 @@ def test_failed_current_file_keeps_successful_body_readable(monkeypatch, async_r
monkeypatch.setattr(reader, "get_relations", AsyncMock(return_value=()))
with pytest.raises(ValueError, match="正文关系"):
async_runner.run(reader.get_text())


@pytest.mark.parametrize(
"resolver,content,expected",
[
("core.markdown.v1", "# 标题\n\n`<node>`", "`<node>`"),
(
"core.html.v1",
'<h1>标题</h1><video src="https://example.org/video">'
'<track src="https://example.org/captions.vtt"></video>',
"https://example.org/captions.vtt",
),
],
)
def test_v2_uses_the_body_format(monkeypatch, async_runner, resolver, content, expected):
from extensions.local_files.resolver import LocalFileV2Resolver
from app.business.info_base.resolver import register_core_resolvers
from app.schemas.info_base.block import BlockModel

register_core_resolvers()
now = datetime.datetime.now(datetime.UTC)
metadata = FileMetadata(
location="inkcre-peer-file://a143b185-f43d-47d1-8ef7-1d3857284b01/Users/a/note.md",
format="markdown",
available=True,
observed_at=now,
)
reader = object.__new__(LocalFileV2Resolver)
monkeypatch.setattr(
reader, "get_raw_content", AsyncMock(return_value=metadata.model_dump_json())
)
monkeypatch.setattr(
reader,
"get_relations",
AsyncMock(return_value=(SimpleNamespace(content="body", to_=20),)),
)
block = BlockModel(
id=20, resolver=resolver, content=content, storage=None, created_at=now, updated_at=now
)
monkeypatch.setattr(BlockService, "get", AsyncMock(return_value=block))
assert expected in async_runner.run(reader.get_text(materialize_missing=False))
legacy = object.__new__(LocalFileResolver)
monkeypatch.setattr(legacy, "get_raw_content", reader.get_raw_content)
monkeypatch.setattr(legacy, "get_relations", reader.get_relations)
with pytest.raises(ValueError, match="类型不符"):
async_runner.run(legacy.get_text())
37 changes: 37 additions & 0 deletions tests/info_base/test_markdown.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
"""Unicode source remains readable without a browser or source Peer."""

import datetime
from unittest.mock import AsyncMock

from app.business.info_base.resolver import (
MarkdownResolver,
ResolverManager,
register_core_resolvers,
)
from app.schemas.info_base.block import BlockModel


def test_markdown_keeps_source_and_unicode(async_runner, monkeypatch):
register_core_resolvers()
assert (
ResolverManager.match_media_type("text/markdown; charset=utf-8") == "core.markdown.v1"
)
source = (
"# 中文\n\n```html\n<node>& value\n indented\n```\n[link](https://example.org)\n"
)
now = datetime.datetime.now(datetime.UTC)
block = BlockModel(
id=1,
resolver="core.markdown.v1",
storage=None,
content=source,
created_at=now,
updated_at=now,
)
reader = ResolverManager.get(block)
assert isinstance(reader, MarkdownResolver)
assert async_runner.run(reader.get_text()) == source
monkeypatch.setattr(
BlockModel, "get_hydrated_content", AsyncMock(return_value=source.encode("utf-16"))
)
assert async_runner.run(reader.get_text(refresh=True)) == source
Loading