feat: Enhance card generation with limitations tracking and logging functionality

This commit is contained in:
Saberlve 2026-04-23 10:22:07 +08:00
parent aaafa78883
commit 8f8c219423
9 changed files with 312 additions and 35 deletions

View File

@ -149,6 +149,7 @@ def create_app(
resolved_client = llm_client or create_card_generation_client( resolved_client = llm_client or create_card_generation_client(
str(llm_payload.get("provider", "deterministic")), str(llm_payload.get("provider", "deterministic")),
str(llm_payload.get("model", "deterministic")), str(llm_payload.get("model", "deterministic")),
log_dir=config.workspace_dir / "logs" / "llm_calls",
) )
except ValueError as exc: except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc raise HTTPException(status_code=400, detail=str(exc)) from exc
@ -257,6 +258,7 @@ def create_app(
resolved_client = llm_client or create_card_generation_client( resolved_client = llm_client or create_card_generation_client(
str(llm_payload.get("provider", "deterministic")), str(llm_payload.get("provider", "deterministic")),
str(llm_payload.get("model", "deterministic")), str(llm_payload.get("model", "deterministic")),
log_dir=config.workspace_dir / "logs" / "llm_calls",
) )
except ValueError as exc: except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc raise HTTPException(status_code=400, detail=str(exc)) from exc

View File

@ -46,6 +46,35 @@ class CardBuilder:
self._update_indexes(item, card_path, source_hash, card_data, language) self._update_indexes(item, card_path, source_hash, card_data, language)
return CardBuildResult(item_key=item.item_key, card_path=card_path, source_hash=source_hash, language=language) return CardBuildResult(item_key=item.item_key, card_path=card_path, source_hash=source_hash, language=language)
# Matches section headings that signal the end of the main body.
# Handles pdftotext artifacts where letters may be spaced apart.
_REFERENCES_PATTERN = re.compile(
r"^(?:"
r"References?|Bibliography|"
r"REFERENCES?|BIBLIOGRAPHY|"
r"R\s+E\s*F\s*E\s*R\s*E\s*N\s*C\s*E\s*S?|"
r"B\s*I\s*B\s*L\s*I\s*O\s*G\s*R\s*A\s*P\s*H\s*Y|"
r"参考文献|"
r"ACKNOWLEDGMENTS?|Acknowledgments?|"
r"A\s*C\s*K\s*N\s*O\s*W\s*L\s*E\s*D\s*G\s*M\s*E\s*N\s*T\s*S?"
r")\s*$",
re.MULTILINE,
)
# Fallback: reference list entries like "[1] A. Author" at the start of a line.
_REFERENCE_ENTRY_PATTERN = re.compile(r"^\[\d+\]\s+[A-Z]", re.MULTILINE)
@classmethod
def _truncate_before_references(cls, text: str) -> str:
match = cls._REFERENCES_PATTERN.search(text)
if match:
return text[: match.start()].rstrip()
# If no heading found, try to detect the first bibliography entry.
entry_match = cls._REFERENCE_ENTRY_PATTERN.search(text)
if entry_match:
return text[: entry_match.start()].rstrip()
return text
def _build_source_bundle(self, item: ZoteroItemRecord, language: str) -> dict[str, object]: def _build_source_bundle(self, item: ZoteroItemRecord, language: str) -> dict[str, object]:
return { return {
"item_key": item.item_key, "item_key": item.item_key,
@ -59,7 +88,10 @@ class CardBuilder:
"collection_paths": item.collection_paths, "collection_paths": item.collection_paths,
"notes": item.notes, "notes": item.notes,
"attachments": item.attachments, "attachments": item.attachments,
"attachment_texts": item.attachment_texts, "attachment_texts": [
self._truncate_before_references(str(text))
for text in item.attachment_texts
],
} }
@staticmethod @staticmethod
@ -104,6 +136,7 @@ class CardBuilder:
("Summary", [str(card_data.get("summary", ""))]), ("Summary", [str(card_data.get("summary", ""))]),
("Core Claims", [str(value) for value in card_data.get("core_claims", [])]), ("Core Claims", [str(value) for value in card_data.get("core_claims", [])]),
("Methods", [str(value) for value in card_data.get("methods", [])]), ("Methods", [str(value) for value in card_data.get("methods", [])]),
("Limitations", [str(value) for value in card_data.get("limitations", [])]),
("Evidence", [str(value) for value in card_data.get("evidence", [])]), ("Evidence", [str(value) for value in card_data.get("evidence", [])]),
("Source Files", self._render_attachments(item.attachments)), ("Source Files", self._render_attachments(item.attachments)),
("Citations", self._render_citations([value for value in card_data.get("citations", []) if isinstance(value, dict)])), ("Citations", self._render_citations([value for value in card_data.get("citations", []) if isinstance(value, dict)])),
@ -204,6 +237,8 @@ class CardBuilder:
"attachments": item.attachments, "attachments": item.attachments,
"keywords": [str(value) for value in card_data.get("keywords", [])], "keywords": [str(value) for value in card_data.get("keywords", [])],
"claims": [str(value) for value in card_data.get("core_claims", [])], "claims": [str(value) for value in card_data.get("core_claims", [])],
"methods": [str(value) for value in card_data.get("methods", [])],
"limitations": [str(value) for value in card_data.get("limitations", [])],
"citations": self._normalize_citations( "citations": self._normalize_citations(
[value for value in card_data.get("citations", []) if isinstance(value, dict)] [value for value in card_data.get("citations", []) if isinstance(value, dict)]
), ),

View File

@ -4,6 +4,8 @@ import json
import os import os
import urllib.request import urllib.request
from collections.abc import Mapping from collections.abc import Mapping
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable, Protocol from typing import Any, Callable, Protocol
@ -28,6 +30,7 @@ class DeterministicCardGenerationClient:
"summary": summary if language == "en" else f"中文:{summary}", "summary": summary if language == "en" else f"中文:{summary}",
"core_claims": claims if language == "en" else [f"中文:{claim}" for claim in claims], "core_claims": claims if language == "en" else [f"中文:{claim}" for claim in claims],
"methods": [], "methods": [],
"limitations": [],
"evidence": attachment_texts[:3], "evidence": attachment_texts[:3],
"citations": [ "citations": [
{ {
@ -59,11 +62,31 @@ class DeepSeekCardGenerationClient:
model: str = "deepseek-chat", model: str = "deepseek-chat",
base_url: str = "https://api.deepseek.com/v1", base_url: str = "https://api.deepseek.com/v1",
request_fn: RequestFunction | None = None, request_fn: RequestFunction | None = None,
log_dir: str | Path | None = None,
) -> None: ) -> None:
self.api_key = api_key self.api_key = api_key
self.model = model self.model = model
self.base_url = base_url.rstrip("/") self.base_url = base_url.rstrip("/")
self.request_fn = request_fn or self._default_request self.request_fn = request_fn or self._default_request
self.log_dir = Path(log_dir) if log_dir else None
def _write_log(self, source_bundle: dict[str, object], payload: dict[str, object], raw_response: dict[str, object], parsed_result: dict[str, object]) -> None:
if self.log_dir is None:
return
self.log_dir.mkdir(parents=True, exist_ok=True)
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S_%f")
item_key = str(source_bundle.get("item_key", "unknown"))
log_path = self.log_dir / f"{timestamp}_{item_key}.json"
log_data = {
"timestamp": datetime.now(timezone.utc).isoformat(),
"item_key": item_key,
"model": self.model,
"source_bundle": source_bundle,
"request_payload": payload,
"raw_response": raw_response,
"parsed_result": parsed_result,
}
log_path.write_text(json.dumps(log_data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def generate_card(self, source_bundle: dict[str, object]) -> dict[str, object]: def generate_card(self, source_bundle: dict[str, object]) -> dict[str, object]:
language = str(source_bundle.get("card_language", "en")) language = str(source_bundle.get("card_language", "en"))
@ -79,30 +102,36 @@ class DeepSeekCardGenerationClient:
{ {
"role": "system", "role": "system",
"content": ( "content": (
"You convert research source bundles into structured evidence cards for academic writing. " "You are a research analyst extracting structured evidence cards from academic papers. "
"Your goal is to produce cards that help a researcher cite this work critically and precisely — "
"not just summarize what the paper claims, but surface the technical method, its assumptions, "
"and its limitations so the card can be used to construct arguments like 'Method X achieves Y, "
"but relies on assumption Z, which fails when...'. "
"Never hallucinate. Use only information explicitly supported by the source bundle. " "Never hallucinate. Use only information explicitly supported by the source bundle. "
"Preserve uncertainty and hedging. Return valid JSON only with keys: " "Preserve uncertainty and hedging. Return valid JSON only with keys: "
"summary, core_claims, methods, evidence, citations, quotable_passages, writing_hints, keywords." "summary, core_claims, methods, limitations, evidence, citations, quotable_passages, writing_hints, keywords."
), ),
}, },
{ {
"role": "user", "role": "user",
"content": ( "content": (
"Given this source bundle, return exactly one JSON object with keys: " "Given this source bundle, return exactly one JSON object with keys: "
"summary, core_claims, methods, evidence, citations, quotable_passages, writing_hints, keywords.\n\n" "summary, core_claims, methods, limitations, evidence, citations, quotable_passages, writing_hints, keywords.\n\n"
"Rules:\n" "Rules:\n"
"1. summary: 2-4 factual sentences, no hype.\n" "1. summary: 3-5 sentences. Describe WHAT the paper does at a high level, WHO it builds on, and WHY the problem matters. Do not list benchmark numbers.\n"
"2. core_claims: concrete findings only; preserve numbers and hedging.\n" "2. core_claims: 3-6 bullet-length claims focused on TECHNICAL CONTRIBUTIONS — novel architectures, key design decisions, theoretical guarantees, or qualitative insights. Avoid 'outperforms X by Y%' unless the number itself is the main contribution. Preserve hedging language.\n"
"3. methods: brief method or setup details actually stated in the source.\n" "3. methods: detailed but concise description of the proposed method. Include: (a) the overall architecture or pipeline, (b) the role of each component and how they interact, (c) key design choices or inductive biases, (d) training or inference procedure if non-standard. If the paper modifies an existing method, state exactly what was changed.\n"
"4. evidence: empirical or textual support from the source, numbers preferred.\n" "4. limitations: an ARRAY of strings. Each string is one explicit limitation, assumption, failure mode, or scope restriction stated or strongly implied by the authors. Also include any methodological gaps you can infer from the source bundle (e.g., 'evaluated only on synthetic data', 'requires expensive online planning', 'assumes a single robot arm'). If none are found, return [\"None explicitly discussed\"].\n"
"5. citations: an array of objects with keys claim, quote, paraphrase, quote_source, use_case.\n" "5. evidence: specific empirical or textual support. Prefer qualitative observations, ablation insights, or case studies over raw benchmark tables. Include numbers only when they directly illustrate a method property (e.g., 'removing component X caused success rate to drop from 78% to 31%').\n"
"6. citations.quote must be verbatim text from notes or attachment_texts only.\n" "6. citations: an array of objects with keys claim, quote, paraphrase, quote_source, use_case.\n"
"7. Return 1-3 citations whenever the source bundle contains usable supporting text in notes or attachment_texts.\n" "7. citations.quote must be verbatim text from notes or attachment_texts only.\n"
"8. quote_source must identify where the quote came from, such as attachment_texts or notes.\n" "8. Return 1-5 citations whenever usable supporting text exists. Prioritize quotes that: (a) describe a method detail, (b) state an assumption or limitation, (c) draw a comparison to prior work.\n"
"9. use_case must be one of direct_quote, paraphrase, or background.\n" "9. quote_source must identify where the quote came from, such as attachment_texts or notes.\n"
"10. quotable_passages: short verbatim excerpts from notes or attachment_texts only.\n" "10. use_case must be one of direct_quote, paraphrase, or background. Mark as 'direct_quote' when the verbatim text is particularly precise or memorable; 'paraphrase' when the idea is useful but the wording is not; 'background' for situational context.\n"
"11. Use empty arrays only when the source bundle truly contains no usable supporting quote; do not fabricate.\n" "11. quotable_passages: 2-5 short verbatim excerpts that would be useful to quote directly in a related-work or methods section. Prioritize passages about technical details, assumptions, or caveats.\n"
f"12. {language_instruction}\n\n" "12. writing_hints: 2-4 sentences suggesting HOW to use this paper in an argument — e.g., 'Cite this when arguing that retrieval-based memory scales better than parametric memory, but note it assumes access to a curated experience buffer.'\n"
"13. Use empty arrays only when the source bundle truly contains no usable supporting quote; do not fabricate.\n"
f"14. {language_instruction}\n\n"
f"Source bundle:\n{json.dumps(source_bundle, ensure_ascii=False)}" f"Source bundle:\n{json.dumps(source_bundle, ensure_ascii=False)}"
), ),
}, },
@ -115,11 +144,12 @@ class DeepSeekCardGenerationClient:
response = self.request_fn(f"{self.base_url}/chat/completions", payload, headers) response = self.request_fn(f"{self.base_url}/chat/completions", payload, headers)
content = str(response["choices"][0]["message"]["content"]) content = str(response["choices"][0]["message"]["content"])
parsed = json.loads(_strip_json_fence(content)) parsed = json.loads(_strip_json_fence(content))
return { result = {
"summary": str(parsed.get("summary", "")), "summary": str(parsed.get("summary", "")),
"core_claims": [str(value) for value in parsed.get("core_claims", [])], "core_claims": _to_str_list(parsed.get("core_claims")),
"methods": [str(value) for value in parsed.get("methods", [])], "methods": _to_str_list(parsed.get("methods")),
"evidence": [str(value) for value in parsed.get("evidence", [])], "limitations": _to_str_list(parsed.get("limitations")),
"evidence": _to_str_list(parsed.get("evidence")),
"citations": [ "citations": [
{ {
"claim": str(value.get("claim", "")), "claim": str(value.get("claim", "")),
@ -131,10 +161,12 @@ class DeepSeekCardGenerationClient:
for value in parsed.get("citations", []) for value in parsed.get("citations", [])
if isinstance(value, dict) if isinstance(value, dict)
], ],
"quotable_passages": [str(value) for value in parsed.get("quotable_passages", [])], "quotable_passages": _to_str_list(parsed.get("quotable_passages")),
"writing_hints": [str(value) for value in parsed.get("writing_hints", [])], "writing_hints": _to_str_list(parsed.get("writing_hints")),
"keywords": [str(value) for value in parsed.get("keywords", [])], "keywords": _to_str_list(parsed.get("keywords")),
} }
self._write_log(source_bundle, payload, response, result)
return result
@staticmethod @staticmethod
def _default_request(url: str, payload: dict[str, object], headers: dict[str, str]) -> dict[str, object]: def _default_request(url: str, payload: dict[str, object], headers: dict[str, str]) -> dict[str, object]:
@ -154,6 +186,7 @@ def create_card_generation_client(
*, *,
env: Mapping[str, str] | None = None, env: Mapping[str, str] | None = None,
request_fn: RequestFunction | None = None, request_fn: RequestFunction | None = None,
log_dir: str | Path | None = None,
) -> CardGenerationClient: ) -> CardGenerationClient:
current_env = os.environ if env is None else env current_env = os.environ if env is None else env
normalized = provider.lower() normalized = provider.lower()
@ -161,10 +194,18 @@ def create_card_generation_client(
api_key = current_env.get("DEEPSEEK_API_KEY") api_key = current_env.get("DEEPSEEK_API_KEY")
if not api_key: if not api_key:
raise ValueError("DEEPSEEK_API_KEY is required when llm provider is deepseek") raise ValueError("DEEPSEEK_API_KEY is required when llm provider is deepseek")
return DeepSeekCardGenerationClient(api_key=api_key, model=model, request_fn=request_fn) return DeepSeekCardGenerationClient(api_key=api_key, model=model, request_fn=request_fn, log_dir=log_dir)
return DeterministicCardGenerationClient() return DeterministicCardGenerationClient()
def _to_str_list(value: object) -> list[str]:
if value is None:
return []
if isinstance(value, str):
return [value] if value.strip() else []
return [str(v) for v in value if v is not None]
def _strip_json_fence(value: str) -> str: def _strip_json_fence(value: str) -> str:
stripped = value.strip() stripped = value.strip()
if stripped.startswith("```"): if stripped.startswith("```"):

View File

@ -47,6 +47,8 @@ class ProjectService:
"summary": card_data.get("summary") if card_data else None, "summary": card_data.get("summary") if card_data else None,
"attachments": card_data.get("attachments", []) if card_data else [], "attachments": card_data.get("attachments", []) if card_data else [],
"claims": card_data.get("claims", []) if card_data else [], "claims": card_data.get("claims", []) if card_data else [],
"methods": card_data.get("methods", []) if card_data else [],
"limitations": card_data.get("limitations", []) if card_data else [],
"citations": card_data.get("citations", []) if card_data else [], "citations": card_data.get("citations", []) if card_data else [],
"quotable_spans": card_data.get("quotable_spans", []) if card_data else [], "quotable_spans": card_data.get("quotable_spans", []) if card_data else [],
} }

View File

@ -530,6 +530,19 @@
.batch-generate-item[aria-disabled="true"] { .batch-generate-item[aria-disabled="true"] {
opacity: 0.72; opacity: 0.72;
} }
.batch-generate-select-all {
display: grid;
grid-template-columns: auto minmax(0, 1fr);
gap: 0.85rem;
align-items: center;
padding: 0.5rem 1.25rem;
border-bottom: 1px solid var(--line);
font-weight: 600;
position: sticky;
top: 0;
background: rgba(255, 253, 248, 0.98);
z-index: 1;
}
.window-scroll { .window-scroll {
min-height: 0; min-height: 0;
overflow: auto; overflow: auto;
@ -737,6 +750,10 @@
<div class="batch-generate-summary"> <div class="batch-generate-summary">
<div id="batch-generate-progress" class="status">未开始批量生成。</div> <div id="batch-generate-progress" class="status">未开始批量生成。</div>
</div> </div>
<label class="batch-generate-select-all">
<input type="checkbox" id="batch-select-all-checkbox" />
<span>全选</span>
</label>
<div id="batch-generate-list" class="batch-generate-list window-scroll collection-items-list"></div> <div id="batch-generate-list" class="batch-generate-list window-scroll collection-items-list"></div>
</div> </div>
</section> </section>
@ -921,6 +938,7 @@
startBatchGenerateButton: document.getElementById("start-batch-generate-button"), startBatchGenerateButton: document.getElementById("start-batch-generate-button"),
batchGenerateProgress: document.getElementById("batch-generate-progress"), batchGenerateProgress: document.getElementById("batch-generate-progress"),
batchGenerateList: document.getElementById("batch-generate-list"), batchGenerateList: document.getElementById("batch-generate-list"),
batchSelectAllCheckbox: document.getElementById("batch-select-all-checkbox"),
refreshProjectsButton: document.getElementById("refresh-projects-button"), refreshProjectsButton: document.getElementById("refresh-projects-button"),
recommendForm: document.getElementById("recommend-form"), recommendForm: document.getElementById("recommend-form"),
recommendStatus: document.getElementById("recommend-status"), recommendStatus: document.getElementById("recommend-status"),
@ -1453,7 +1471,7 @@
} }
function canGenerateProjectItem(status) { function canGenerateProjectItem(status) {
return status === "pending" || status === "failed"; return status === "pending" || status === "failed" || status === "done";
} }
function canGenerateProjectItemInBatch(status) { function canGenerateProjectItemInBatch(status) {
@ -1513,9 +1531,11 @@
} }
elements.batchGenerateList.innerHTML = ""; elements.batchGenerateList.innerHTML = "";
const selectableKeys = [];
for (const item of state.currentProjectItems) { for (const item of state.currentProjectItems) {
const status = getDisplayedProjectItemStatus(item); const status = getDisplayedProjectItemStatus(item);
const selectable = status !== "done" && status !== "generating"; const selectable = status !== "generating";
if (selectable) selectableKeys.push(item.item_key);
const row = document.createElement("label"); const row = document.createElement("label");
row.className = "project-item batch-generate-item"; row.className = "project-item batch-generate-item";
row.setAttribute("aria-disabled", selectable ? "false" : "true"); row.setAttribute("aria-disabled", selectable ? "false" : "true");
@ -1538,6 +1558,12 @@
}); });
elements.batchGenerateList.appendChild(row); elements.batchGenerateList.appendChild(row);
} }
// Sync select-all checkbox state
const selectedCount = selectableKeys.filter((k) => state.batchGenerateSelectedKeys.has(k)).length;
if (elements.batchSelectAllCheckbox) {
elements.batchSelectAllCheckbox.checked = selectableKeys.length > 0 && selectedCount === selectableKeys.length;
elements.batchSelectAllCheckbox.indeterminate = selectedCount > 0 && selectedCount < selectableKeys.length;
}
updateBatchGenerateControls(); updateBatchGenerateControls();
} }
@ -1671,6 +1697,7 @@
if (status === "done") { if (status === "done") {
const attachments = renderProjectItemAttachments(item.attachments || []); const attachments = renderProjectItemAttachments(item.attachments || []);
const claims = (item.claims || []).map((claim) => `<li>${escapeHtml(claim)}</li>`).join("") || "<li>暂无 claims</li>"; const claims = (item.claims || []).map((claim) => `<li>${escapeHtml(claim)}</li>`).join("") || "<li>暂无 claims</li>";
const limitations = (item.limitations || []).map((lim) => `<li>${escapeHtml(lim)}</li>`).join("") || "";
const citations = renderProjectItemCitations(item.citations || []); const citations = renderProjectItemCitations(item.citations || []);
detail.innerHTML = ` detail.innerHTML = `
<div class="meta">${item.item_key}</div> <div class="meta">${item.item_key}</div>
@ -1679,6 +1706,7 @@
${attachments} ${attachments}
<h4>Claims</h4> <h4>Claims</h4>
<ul>${claims}</ul> <ul>${claims}</ul>
${limitations ? `<h4>Limitations</h4><ul>${limitations}</ul>` : ""}
<h4>Citations</h4> <h4>Citations</h4>
${citations} ${citations}
`; `;
@ -2189,6 +2217,21 @@
elements.closeBatchGenerateButton.addEventListener("click", closeBatchGenerateWindow); elements.closeBatchGenerateButton.addEventListener("click", closeBatchGenerateWindow);
elements.selectPendingBatchButton.addEventListener("click", selectAllPendingBatchItems); elements.selectPendingBatchButton.addEventListener("click", selectAllPendingBatchItems);
elements.clearBatchSelectionButton.addEventListener("click", clearBatchGenerateSelection); elements.clearBatchSelectionButton.addEventListener("click", clearBatchGenerateSelection);
if (elements.batchSelectAllCheckbox) {
elements.batchSelectAllCheckbox.addEventListener("change", () => {
const checked = elements.batchSelectAllCheckbox.checked;
for (const item of state.currentProjectItems) {
const status = getDisplayedProjectItemStatus(item);
if (status === "generating") continue;
if (checked) {
state.batchGenerateSelectedKeys.add(item.item_key);
} else {
state.batchGenerateSelectedKeys.delete(item.item_key);
}
}
renderBatchGenerateList();
});
}
elements.startBatchGenerateButton.addEventListener("click", () => { elements.startBatchGenerateButton.addEventListener("click", () => {
runBatchGenerate().catch(() => null); runBatchGenerate().catch(() => null);
}); });

View File

@ -401,10 +401,11 @@ class ZoteroReader:
def _read_attachment_records(self, conn: sqlite3.Connection, item_id: int) -> list[dict[str, object]]: def _read_attachment_records(self, conn: sqlite3.Connection, item_id: int) -> list[dict[str, object]]:
rows = conn.execute( rows = conn.execute(
""" """
select contentType, path select ia.contentType, ia.path, i.key as itemKey
from itemAttachments from itemAttachments ia
where parentItemID = ? join items i on ia.itemID = i.itemID
order by itemID asc where ia.parentItemID = ?
order by ia.itemID asc
""", """,
(item_id,), (item_id,),
) )
@ -412,7 +413,8 @@ class ZoteroReader:
for row in rows: for row in rows:
raw_path = str(row["path"] or "") raw_path = str(row["path"] or "")
content_type = str(row["contentType"] or "") content_type = str(row["contentType"] or "")
attachment_path = self._resolve_attachment_path(raw_path) item_key = str(row["itemKey"] or "")
attachment_path = self._resolve_attachment_path(raw_path, item_key)
if attachment_path is None: if attachment_path is None:
continue continue
text = self._extract_attachment_text_from_resolved_path(attachment_path, content_type) text = self._extract_attachment_text_from_resolved_path(attachment_path, content_type)
@ -451,9 +453,16 @@ class ZoteroReader:
return attachment_path.read_text(encoding="utf-8", errors="ignore").strip() return attachment_path.read_text(encoding="utf-8", errors="ignore").strip()
def _resolve_attachment_path(self, raw_path: str) -> Path | None: def _resolve_attachment_path(self, raw_path: str, item_key: str = "") -> Path | None:
if raw_path.startswith("storage:"): if raw_path.startswith("storage:"):
relative_path = raw_path.removeprefix("storage:") relative_path = raw_path.removeprefix("storage:")
# Zotero storage paths come in two forms:
# storage:<key>/filename.pdf -> storage/<key>/filename.pdf
# storage:filename.pdf -> storage/<attachment_item_key>/filename.pdf
if "/" in relative_path:
return self.zotero_data_dir / "storage" / relative_path
if item_key:
return self.zotero_data_dir / "storage" / item_key / relative_path
return self.zotero_data_dir / "storage" / relative_path return self.zotero_data_dir / "storage" / relative_path
path = Path(raw_path) path = Path(raw_path)
if path.is_absolute(): if path.is_absolute():

View File

@ -58,9 +58,12 @@ def build_fixture_zotero_dir(zotero_dir: Path) -> None:
(3, "2024"), (3, "2024"),
], ],
) )
cur.execute( cur.executemany(
"insert into items(itemID, itemTypeID, dateAdded, dateModified, clientDateModified, libraryID, key, version, synced) values (?, ?, '', '', '', 1, ?, 1, 1)", "insert into items(itemID, itemTypeID, dateAdded, dateModified, clientDateModified, libraryID, key, version, synced) values (?, ?, '', '', '', 1, ?, 1, 1)",
[
(1, 1, "PAPER0001"), (1, 1, "PAPER0001"),
(3, 3, "ATTACH01"),
],
) )
cur.executemany( cur.executemany(
"insert into itemData(itemID, fieldID, valueID) values (?, ?, ?)", "insert into itemData(itemID, fieldID, valueID) values (?, ?, ?)",

View File

@ -0,0 +1,142 @@
"""End-to-end test: verify limitations flow through the entire pipeline."""
from __future__ import annotations
import json
from pathlib import Path
from zotero_kb.cards import CardBuilder
from zotero_kb.config import AppConfig
from zotero_kb.llm import CardGenerationClient
from zotero_kb.projects import ProjectService
from zotero_kb.workspace import Workspace
from zotero_kb.zotero_reader import ZoteroItemRecord
class FakeLlmClientWithLimitations:
"""Returns card data including limitations."""
def generate_card(self, source_bundle: dict[str, object]) -> dict[str, object]:
print("\n" + "=" * 60)
print("STEP 1: Source bundle received by LLM client")
print("=" * 60)
print(json.dumps(source_bundle, ensure_ascii=False, indent=2))
language = str(source_bundle.get("card_language", "en"))
limitations = [
"Assumes all demonstration trajectories are successful and noise-free.",
"Requires an expensive online planning step at test time.",
"Evaluated only on a single robotic manipulation domain with limited diversity.",
] if language == "en" else [
"假设所有演示轨迹都是成功且无噪声的。",
"在测试时需要昂贵的在线规划步骤。",
"仅在单一机器人操作领域上评估,多样性有限。",
]
result = {
"summary": "A method for robot control using hierarchical policies." if language == "en" else "一种使用分层策略进行机器人控制的方法。",
"core_claims": ["Claim A", "Claim B"],
"methods": ["Hierarchical policy framework", "Experience retrieval mechanism"],
"limitations": limitations,
"evidence": ["Ablation shows 78% → 31% drop"],
"citations": [
{
"claim": "Retrieval scales better than parametric memory.",
"quote": "Our retrieval-based approach outperforms...",
"paraphrase": "Retrieval works better.",
"quote_source": "attachment_texts",
"use_case": "direct_quote",
}
],
"quotable_passages": ["Short excerpt about retrieval."],
"writing_hints": ["Cite when arguing about retrieval vs parametric."],
"keywords": ["robotics", "memory"],
}
print("\n" + "=" * 60)
print("STEP 2: LLM response (parsed JSON)")
print("=" * 60)
print(json.dumps(result, ensure_ascii=False, indent=2))
return result
def test_limitations_end_to_end(tmp_path: Path) -> None:
workspace_dir = tmp_path / "workspace"
workspace_dir.mkdir()
# Setup workspace + project
config = AppConfig(workspace_dir=workspace_dir, zotero_data_dir=tmp_path / "zotero")
workspace = Workspace(config)
project = workspace.create_project(
project_id="test-project",
name="Test Project",
llm_provider="deepseek",
llm_model="deepseek-chat",
card_language="en",
)
project_id = project.project_id
# Build a fake item
item = ZoteroItemRecord(
item_key="MEMER2025",
title="MemER: Scaling Up Memory for Robot Control",
creators=["Sridhar et al."],
year=2025,
item_type="journalArticle",
abstract="We propose MemER, a hierarchical policy framework.",
tags=["robotics", "memory"],
collection_paths=[["Robotics", "Manipulation"]],
notes=["Important note about method."],
attachments=[
{
"filename": "paper.pdf",
"path": "/tmp/paper.pdf",
"content_type": "application/pdf",
"is_pdf": True,
}
],
attachment_texts=[
"Our retrieval-based approach outperforms prior parametric methods...",
"Limitations: We assume noise-free demonstrations.",
],
)
# STEP 3: CardBuilder generates the card
print("\n" + "=" * 60)
print("STEP 3: CardBuilder.build_or_update()")
print("=" * 60)
builder = CardBuilder(workspace_dir, FakeLlmClientWithLimitations())
result = builder.build_or_update(item, language="en")
# STEP 4: Read generated Markdown
print("\n" + "=" * 60)
print("STEP 4: Generated Markdown card")
print("=" * 60)
card_md = result.card_path.read_text(encoding="utf-8")
print(card_md)
# Verify Markdown contains Limitations
assert "# Limitations" in card_md
assert "noise-free" in card_md
assert "online planning" in card_md
# STEP 5: ProjectService rebuilds project-index
print("\n" + "=" * 60)
print("STEP 5: ProjectService._rebuild_project_index()")
print("=" * 60)
ps = ProjectService(workspace_dir)
ps.add_items(project_id, ["MEMER2025"])
project_index = ps.get_project_view(project_id)
print(json.dumps(project_index, ensure_ascii=False, indent=2))
# Verify project-index carries limitations
items = project_index["items"]
assert len(items) == 1
project_item = items[0]
assert "limitations" in project_item
assert len(project_item["limitations"]) == 3
assert "noise-free" in str(project_item["limitations"])
print("\n" + "=" * 60)
print("ALL CHECKS PASSED")
print("=" * 60)

View File

@ -67,7 +67,7 @@ def test_deepseek_client_uses_env_api_key_and_parses_json() -> None:
assert "locator" not in result["citations"][0] assert "locator" not in result["citations"][0]
assert captured["url"] == "https://api.deepseek.com/v1/chat/completions" assert captured["url"] == "https://api.deepseek.com/v1/chat/completions"
assert captured["payload"]["model"] == "deepseek-chat" assert captured["payload"]["model"] == "deepseek-chat"
assert "Return 1-3 citations whenever the source bundle contains usable supporting text" in captured["payload"]["messages"][1]["content"] assert "Return 1-5 citations whenever usable supporting text exists" in captured["payload"]["messages"][1]["content"]
assert "quote_source must identify where the quote came from" in captured["payload"]["messages"][1]["content"] assert "quote_source must identify where the quote came from" in captured["payload"]["messages"][1]["content"]
assert "Write summary, core_claims, methods, evidence, paraphrase fields, and writing_hints in Chinese." in captured["payload"]["messages"][1]["content"] assert "Write summary, core_claims, methods, evidence, paraphrase fields, and writing_hints in Chinese." in captured["payload"]["messages"][1]["content"]
assert captured["headers"]["Authorization"] == "Bearer sk-test" assert captured["headers"]["Authorization"] == "Bearer sk-test"