{
  "artifact": "brainbox-synthetic-v1-summary",
  "artifact_purpose": "Downloadable summary of the fresh-rebuild synthetic retrieval baseline: method, raw counts and limits. Stripped of local filesystem paths; contains no corpus text and no personal data.",
  "benchmark": "brainbox-synthetic-v1",
  "run_date": "2026-09-30",
  "status": "Frozen baseline. Not tuned, not upgraded; held-out improvement remains open work.",
  "scope": {
    "what_it_is": "Retrieval-only evaluation of the fresh brainbox rebuild on a frozen synthetic corpus of 30 fictitious memories, using real cached bge-m3 vectors and the actual PostgreSQL verbs against a local Docker harness.",
    "what_it_is_not": [
      "NOT a generated-answer evaluation (no answers were judged)",
      "NOT hosted-service latency (no network inference, no cloud transport)",
      "NOT real user data (all content is fictitious)",
      "NOT comparable to the archived LongMemEval judged-answer accuracy from the old deployment (different metric type, different deployment)",
      "NOT a competitor comparison (no other system was run)"
    ]
  },
  "method": {
    "embedding": "@cf/baai/bge-m3 vectors, pre-fetched once into a cache and resolved in-process during the run (no model inference at benchmark time)",
    "retrieval": "hybrid vector + lexical SQL against PostgreSQL 17 with pgvector in a task-owned Docker container; RRF fusion (k=60); recall limit 5",
    "dataset": "30 memories, 30 known-answer queries (types: exact 6, paraphrase 12, multi_source 6, time 6), 10 unanswerable queries (5 near-topic, 5 out-of-domain), 3 lifecycle memories",
    "thresholds_frozen": {
      "abstain_vector_sim_floor": 0.68,
      "abstain_ts_rank_floor": 0.01,
      "rrf_k": 60,
      "vector_candidates": 48,
      "recall_limit": 5
    },
    "seeds": "none - the dataset is frozen; item ids are random UUIDs mapped through frozen aliases",
    "independent_recalculation": "A separate Python pass recomputed headline counts from the raw cases (set intersections and reciprocal ranks, not the benchmark math helper) and matched."
  },
  "results_known_answer": {
    "relevant_item_recall_at_5": { "numerator": 26, "denominator": 40, "value": 0.65 },
    "queries_with_at_least_one_relevant_hit": { "numerator": 22, "denominator": 30 },
    "mrr": 0.6261,
    "macro_mean_recall_at_5": 0.7167,
    "abstained_queries": 2,
    "degraded_or_errored": 0,
    "by_type": {
      "exact":        { "queries": 6, "relevant_items": 6,  "relevant_retrieved_at_5": 6, "queries_with_hit": 6, "mrr": 1.0 },
      "paraphrase":   { "queries": 12, "relevant_items": 12, "relevant_retrieved_at_5": 7, "queries_with_hit": 7, "mrr": 0.4542 },
      "multi_source": { "queries": 6, "relevant_items": 16, "relevant_retrieved_at_5": 7, "queries_with_hit": 3, "mrr": 0.4167 },
      "time":         { "queries": 6, "relevant_items": 6,  "relevant_retrieved_at_5": 6, "queries_with_hit": 6, "mrr": 0.8056 }
    }
  },
  "results_unanswerable": {
    "n": 10,
    "returned_results_flagged_weak_or_partial": 7,
    "fully_abstained_empty": 3,
    "note": "All 7 non-empty returns were near-topic or generic matches carrying weak/partial notes. Retrieval returning a related memory about an entity does not establish a missing attribute - e.g. asking for gym opening hours at a building returns other memories about that building, not the missing attribute. Abstention flags and empty result sets are different measures."
  },
  "lifecycle_and_isolation": {
    "assertions_passed": 17,
    "assertions_failed": 0,
    "unscored_observations": 1,
    "coverage": [
      "correction replaces stored text; old unique wording stops being the only path to the item; new wording retrieves it",
      "forget hard-deletes rows; forgotten items are absent from results; re-remembering tombstoned text is rejected",
      "stale-snapshot restore does not resurrect corrected-away or forgotten content; untouched baseline items restore intact with vectors",
      "tenant isolation: cross-tenant connects rejected both directions; wrong password denied; owning role connects to its own database"
    ],
    "unscored_observation_detail": "REPORT-ONLY: after correction, the old-wording query still ranked the item first via vector similarity - a semantic-drift quality observation, not a scored pass/fail."
  },
  "latency_local_only": {
    "caveat": "LOCAL Docker Postgres + cached-embedding execution only. NOT end-to-end production latency: production adds embedding inference over the network and cloud database transport round trips.",
    "recall_ms":  { "n": 47, "p50": 1.96, "p95": 3.11, "max": 5.10 },
    "remember_write_ms": { "n": 33, "p50": 2.69, "p95": 6.86, "max": 8.07 },
    "embed_cache_lookups": 81,
    "max_embed_cache_lookup_ns": 132667
  },
  "environment": {
    "node": "v26.7.0",
    "platform": "darwin/arm64",
    "docker_image": "pgvector/pgvector:pg17",
    "postgres": "PostgreSQL 17.11",
    "sql_adapter": "node-pg direct",
    "harness": "task-owned Docker container; no shared live database"
  },
  "provenance": {
    "evidence_files_supplied_by_root_2026-09-30": {
      "benchmark-baseline-original-report.json": "sha256 1793957c3b3deb8daa06b86ea876929a2479f564aaf477e344f71c3ef6b4dfd7",
      "benchmark-baseline-original-receipt.json": "sha256 8ed821965b2eeae54ccebe91e28fad1f987ea55db3953f1b66edd3750b612e80 (fixture sha256 615471029550ec349a21a64619a6ce067ff1ab4f090e746979bb6fd5acff8aa2; embedding cache sha256 7138131d7b55bff8dbe6fbedcb07cb48440af5094f0b7577fa9ca945b5e31c95; source revision b598602698011c86aa1044eb5a56dbc82bcd2e15, tree not clean at run time)",
      "benchmark-reporting-correction.json": "sha256 b9427ccdaaa4816cb79c64c394bf0b3e0bdea454a7f0a23db9a8e8537c36046b",
      "benchmark-independent-recalculation.json": "sha256 f9e4001dec01f853491ac0bf78f84cbff8cc2ca4e455c8312d2e01a39df280fb"
    },
    "independent_review": "Pending. Root supplied and inspected the evidence; independent re-review of this baseline had not completed at publication of this summary."
  },
  "limitations": [
    "Synthetic fictitious corpus; results say nothing about retrieval quality on real user memories beyond this frozen dataset.",
    "Latency excludes production network inference and cloud transport.",
    "No historical AWS/Neo4j empirical comparison exists for brainbox; none is fabricated.",
    "No competitor system was run; no comparison score exists.",
    "Thresholds inherited from the legacy calibration were frozen for this run; no tuning was performed on this benchmark.",
    "The set is tiny (30 known queries, 40 relevant items); uncertainty is not computed and no confidence interval is implied.",
    "Held-out evaluation and retrieval improvements remain open; this baseline must not be re-labelled as a tuned score."
  ]
}