143 lines
5.3 KiB
Python
143 lines
5.3 KiB
Python
"""End-to-end test: verify limitations flow through the entire pipeline."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from zotero_kb.cards import CardBuilder
|
|
from zotero_kb.config import AppConfig
|
|
from zotero_kb.llm import CardGenerationClient
|
|
from zotero_kb.projects import ProjectService
|
|
from zotero_kb.workspace import Workspace
|
|
from zotero_kb.zotero_reader import ZoteroItemRecord
|
|
|
|
|
|
class FakeLlmClientWithLimitations:
|
|
"""Returns card data including limitations."""
|
|
|
|
def generate_card(self, source_bundle: dict[str, object]) -> dict[str, object]:
|
|
print("\n" + "=" * 60)
|
|
print("STEP 1: Source bundle received by LLM client")
|
|
print("=" * 60)
|
|
print(json.dumps(source_bundle, ensure_ascii=False, indent=2))
|
|
|
|
language = str(source_bundle.get("card_language", "en"))
|
|
limitations = [
|
|
"Assumes all demonstration trajectories are successful and noise-free.",
|
|
"Requires an expensive online planning step at test time.",
|
|
"Evaluated only on a single robotic manipulation domain with limited diversity.",
|
|
] if language == "en" else [
|
|
"假设所有演示轨迹都是成功且无噪声的。",
|
|
"在测试时需要昂贵的在线规划步骤。",
|
|
"仅在单一机器人操作领域上评估，多样性有限。",
|
|
]
|
|
|
|
result = {
|
|
"summary": "A method for robot control using hierarchical policies." if language == "en" else "一种使用分层策略进行机器人控制的方法。",
|
|
"core_claims": ["Claim A", "Claim B"],
|
|
"methods": ["Hierarchical policy framework", "Experience retrieval mechanism"],
|
|
"limitations": limitations,
|
|
"evidence": ["Ablation shows 78% → 31% drop"],
|
|
"citations": [
|
|
{
|
|
"claim": "Retrieval scales better than parametric memory.",
|
|
"quote": "Our retrieval-based approach outperforms...",
|
|
"paraphrase": "Retrieval works better.",
|
|
"quote_source": "attachment_texts",
|
|
"use_case": "direct_quote",
|
|
}
|
|
],
|
|
"quotable_passages": ["Short excerpt about retrieval."],
|
|
"writing_hints": ["Cite when arguing about retrieval vs parametric."],
|
|
"keywords": ["robotics", "memory"],
|
|
}
|
|
|
|
print("\n" + "=" * 60)
|
|
print("STEP 2: LLM response (parsed JSON)")
|
|
print("=" * 60)
|
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
|
return result
|
|
|
|
|
|
def test_limitations_end_to_end(tmp_path: Path) -> None:
|
|
workspace_dir = tmp_path / "workspace"
|
|
workspace_dir.mkdir()
|
|
|
|
# Setup workspace + project
|
|
config = AppConfig(workspace_dir=workspace_dir, zotero_data_dir=tmp_path / "zotero")
|
|
workspace = Workspace(config)
|
|
project = workspace.create_project(
|
|
project_id="test-project",
|
|
name="Test Project",
|
|
llm_provider="deepseek",
|
|
llm_model="deepseek-chat",
|
|
card_language="en",
|
|
)
|
|
project_id = project.project_id
|
|
|
|
# Build a fake item
|
|
item = ZoteroItemRecord(
|
|
item_key="MEMER2025",
|
|
title="MemER: Scaling Up Memory for Robot Control",
|
|
creators=["Sridhar et al."],
|
|
year=2025,
|
|
item_type="journalArticle",
|
|
abstract="We propose MemER, a hierarchical policy framework.",
|
|
tags=["robotics", "memory"],
|
|
collection_paths=[["Robotics", "Manipulation"]],
|
|
notes=["Important note about method."],
|
|
attachments=[
|
|
{
|
|
"filename": "paper.pdf",
|
|
"path": "/tmp/paper.pdf",
|
|
"content_type": "application/pdf",
|
|
"is_pdf": True,
|
|
}
|
|
],
|
|
attachment_texts=[
|
|
"Our retrieval-based approach outperforms prior parametric methods...",
|
|
"Limitations: We assume noise-free demonstrations.",
|
|
],
|
|
)
|
|
|
|
# STEP 3: CardBuilder generates the card
|
|
print("\n" + "=" * 60)
|
|
print("STEP 3: CardBuilder.build_or_update()")
|
|
print("=" * 60)
|
|
builder = CardBuilder(workspace_dir, FakeLlmClientWithLimitations())
|
|
result = builder.build_or_update(item, language="en")
|
|
|
|
# STEP 4: Read generated Markdown
|
|
print("\n" + "=" * 60)
|
|
print("STEP 4: Generated Markdown card")
|
|
print("=" * 60)
|
|
card_md = result.card_path.read_text(encoding="utf-8")
|
|
print(card_md)
|
|
|
|
# Verify Markdown contains Limitations
|
|
assert "# Limitations" in card_md
|
|
assert "noise-free" in card_md
|
|
assert "online planning" in card_md
|
|
|
|
# STEP 5: ProjectService rebuilds project-index
|
|
print("\n" + "=" * 60)
|
|
print("STEP 5: ProjectService._rebuild_project_index()")
|
|
print("=" * 60)
|
|
ps = ProjectService(workspace_dir)
|
|
ps.add_items(project_id, ["MEMER2025"])
|
|
project_index = ps.get_project_view(project_id)
|
|
|
|
print(json.dumps(project_index, ensure_ascii=False, indent=2))
|
|
|
|
# Verify project-index carries limitations
|
|
items = project_index["items"]
|
|
assert len(items) == 1
|
|
project_item = items[0]
|
|
assert "limitations" in project_item
|
|
assert len(project_item["limitations"]) == 3
|
|
assert "noise-free" in str(project_item["limitations"])
|
|
|
|
print("\n" + "=" * 60)
|
|
print("ALL CHECKS PASSED")
|
|
print("=" * 60)
|