"""End-to-end test: verify limitations flow through the entire pipeline.""" from __future__ import annotations import json from pathlib import Path from zotero_kb.cards import CardBuilder from zotero_kb.config import AppConfig from zotero_kb.llm import CardGenerationClient from zotero_kb.projects import ProjectService from zotero_kb.workspace import Workspace from zotero_kb.zotero_reader import ZoteroItemRecord class FakeLlmClientWithLimitations: """Returns card data including limitations.""" def generate_card(self, source_bundle: dict[str, object]) -> dict[str, object]: print("\n" + "=" * 60) print("STEP 1: Source bundle received by LLM client") print("=" * 60) print(json.dumps(source_bundle, ensure_ascii=False, indent=2)) language = str(source_bundle.get("card_language", "en")) limitations = [ "Assumes all demonstration trajectories are successful and noise-free.", "Requires an expensive online planning step at test time.", "Evaluated only on a single robotic manipulation domain with limited diversity.", ] if language == "en" else [ "假设所有演示轨迹都是成功且无噪声的。", "在测试时需要昂贵的在线规划步骤。", "仅在单一机器人操作领域上评估,多样性有限。", ] result = { "summary": "A method for robot control using hierarchical policies." if language == "en" else "一种使用分层策略进行机器人控制的方法。", "core_claims": ["Claim A", "Claim B"], "methods": ["Hierarchical policy framework", "Experience retrieval mechanism"], "limitations": limitations, "evidence": ["Ablation shows 78% → 31% drop"], "citations": [ { "claim": "Retrieval scales better than parametric memory.", "quote": "Our retrieval-based approach outperforms...", "paraphrase": "Retrieval works better.", "quote_source": "attachment_texts", "use_case": "direct_quote", } ], "quotable_passages": ["Short excerpt about retrieval."], "writing_hints": ["Cite when arguing about retrieval vs parametric."], "keywords": ["robotics", "memory"], } print("\n" + "=" * 60) print("STEP 2: LLM response (parsed JSON)") print("=" * 60) print(json.dumps(result, ensure_ascii=False, indent=2)) return result def test_limitations_end_to_end(tmp_path: Path) -> None: workspace_dir = tmp_path / "workspace" workspace_dir.mkdir() # Setup workspace + project config = AppConfig(workspace_dir=workspace_dir, zotero_data_dir=tmp_path / "zotero") workspace = Workspace(config) project = workspace.create_project( project_id="test-project", name="Test Project", llm_provider="deepseek", llm_model="deepseek-chat", card_language="en", ) project_id = project.project_id # Build a fake item item = ZoteroItemRecord( item_key="MEMER2025", title="MemER: Scaling Up Memory for Robot Control", creators=["Sridhar et al."], year=2025, item_type="journalArticle", abstract="We propose MemER, a hierarchical policy framework.", tags=["robotics", "memory"], collection_paths=[["Robotics", "Manipulation"]], notes=["Important note about method."], attachments=[ { "filename": "paper.pdf", "path": "/tmp/paper.pdf", "content_type": "application/pdf", "is_pdf": True, } ], attachment_texts=[ "Our retrieval-based approach outperforms prior parametric methods...", "Limitations: We assume noise-free demonstrations.", ], ) # STEP 3: CardBuilder generates the card print("\n" + "=" * 60) print("STEP 3: CardBuilder.build_or_update()") print("=" * 60) builder = CardBuilder(workspace_dir, FakeLlmClientWithLimitations()) result = builder.build_or_update(item, language="en") # STEP 4: Read generated Markdown print("\n" + "=" * 60) print("STEP 4: Generated Markdown card") print("=" * 60) card_md = result.card_path.read_text(encoding="utf-8") print(card_md) # Verify Markdown contains Limitations assert "# Limitations" in card_md assert "noise-free" in card_md assert "online planning" in card_md # STEP 5: ProjectService rebuilds project-index print("\n" + "=" * 60) print("STEP 5: ProjectService._rebuild_project_index()") print("=" * 60) ps = ProjectService(workspace_dir) ps.add_items(project_id, ["MEMER2025"]) project_index = ps.get_project_view(project_id) print(json.dumps(project_index, ensure_ascii=False, indent=2)) # Verify project-index carries limitations items = project_index["items"] assert len(items) == 1 project_item = items[0] assert "limitations" in project_item assert len(project_item["limitations"]) == 3 assert "noise-free" in str(project_item["limitations"]) print("\n" + "=" * 60) print("ALL CHECKS PASSED") print("=" * 60)