\documentclass[11pt]{article}
\usepackage[margin=1in]{geometry}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{booktabs}
\usepackage{graphicx}
\usepackage{float}
\usepackage{hyperref}

\title{OpenClawBrain v12.2.6+: Shadow Routing with QTsim Confidence Mixing and Unified Policy Learning}
\author{Jonathan Gu}
\date{March 2, 2026}

\begin{document}
\maketitle

\noindent\textbf{Version: v12.2.6+}

\section{Abstract}
OpenClawBrain is a TypeScript-first package workspace for retrieval routing in long-lived agents, deployed behind an OpenClaw-owned runtime boundary. The system is built around a strict hot/cold path split: the hot path is local and bounded for latency-critical routing, while the cold path runs asynchronous learning and structure updates.
We define a \emph{turn} as a single LLM API call. End-to-end cost is $C=\sum_{i=1}^{T} \mathrm{tokens}_i \cdot \mathrm{price}_i$, and the system targets lower $T$ (fewer calls per user-visible request) and lower per-call token counts on the hot path.
The current system uses two route signals: a structural prior ($\mathrm{graph\_prior}$) and a query-time compatibility signal (QTsim). A confidence mixer combines those signals using uncertainty features, including entropy and margin, to decide how much weight each source gets per decision.
Learning is performed through teacher distillation plus policy-gradient updates under a reward-source hierarchy. Human corrections have highest authority, then self-learning outcomes, then harvested signals, then high-volume teacher labels.
This release also starts RL-native maintenance of graph structure: Phase 2a is shipped and follow-on phases cover connect/split/merge actions under policy control.

\section{1. System Overview}
OpenClawBrain is designed around a strict split, with package contracts and learning components in OpenClawBrain and runtime orchestration in OpenClaw:
\begin{itemize}
  \item \textbf{Hot path:} local embedding retrieval and bounded traversal only.
  \item \textbf{Cold path:} asynchronous learning and structure updates.
\end{itemize}
We define a \emph{turn} as a single LLM API call. Total cost can be written as
\[
C=\sum_{i=1}^{T} \mathrm{tokens}_i \cdot \mathrm{price}_i
\]
and OpenClawBrain reduces $T$ and per-call token counts by keeping routing local and context bounded on the hot path, while allowing optional teacher calls only in the cold path.

\begin{figure}[t]
\centering
\includegraphics[width=0.76\textwidth]{architecture-vertical.jpg}
\caption{System diagram: latency-critical hot path stays local and bounded, while shadow replay and policy updates run asynchronously in the cold path.}
\label{fig:system-diagram}
\end{figure}

\subsection*{1.1 Routing Inputs}
At each route decision over candidate targets $\mathcal{A}(s)$ from state $s$:
\begin{itemize}
  \item $\mathrm{graph\_prior}(a\mid s)$ captures persistent graph structure.
  \item $\mathrm{QTsim}(a\mid q,s)$ captures query-conditioned fit.
\end{itemize}

\subsection*{1.2 Confidence Mixing}
The router computes a query-local mix coefficient $\lambda \in [0,1]$ from confidence features.
Two required features are:
\begin{itemize}
  \item \textbf{Entropy} of QTsim over candidates (high entropy means low confidence).
  \item \textbf{Margin} between top-1 and top-2 QTsim scores (low margin means ambiguity).
\end{itemize}
A simple form is:
\[
\lambda = \sigma\!\big(\alpha_0 + \alpha_1\,\mathrm{margin}_{\mathrm{QTsim}} - \alpha_2\,H(\mathrm{QTsim})\big)
\]
\[
\pi(a\mid q,s) = \lambda\,\mathrm{QTsim}(a\mid q,s) + (1-\lambda)\,\mathrm{graph\_prior}(a\mid s)
\]
where $\sigma$ is a logistic gate and $H$ is entropy.
When QTsim is sharp and separated, $\lambda$ increases. When QTsim is uncertain, the policy falls back toward graph prior.

\section{2. Learning Loop: Distillation + Policy Gradient}
The learning loop combines distillation-style supervision with policy-gradient refinement.

\subsection*{2.1 Teacher Distillation}
Offline replay samples route decisions and asks a teacher policy for preferred actions (or distributions). Distillation improves route quality quickly and provides dense supervision for states with sparse human labels.

\subsection*{2.2 Policy-Gradient Update}
Given route trajectory $\tau=(s_0,a_0,\dots,s_T,a_T)$ and scalar return $R(\tau)$:
\[
\nabla J(\theta) = \mathbb{E}_{\tau\sim\pi_\theta}\left[\sum_{t=0}^{T} (R(\tau)-b_t)\nabla_\theta \log \pi_\theta(a_t\mid s_t)\right]
\]
where $b_t$ is a baseline. Distillation targets and reward-weighted gradients are both applied; the exact blend is implementation-defined per run mode.

\subsection*{2.3 Reward-Source Hierarchy}
The system resolves conflicting supervision by ordered authority:
\begin{enumerate}
  \item Human correction/approval (highest authority)
  \item Self-learning outcomes from deployed agents
  \item Harvested weak signals
  \item Teacher labels from async replay (lowest per-sample authority, highest volume)
\end{enumerate}
This hierarchy constrains updates so high-authority signals can override lower-level noise.

\section{3. RL-Native Maintain (Structure Learning)}
OpenClawBrain treats structure maintenance as a control problem over graph actions.

\subsection*{3.1 Phase 2a (Shipped)}
Phase 2a ships RL-native maintenance controls that integrate policy outcomes with structure scoring and safe-apply gates. In practice, this means structure candidates are generated, scored with policy-derived signals, and applied only when guardrails pass.

\subsection*{3.2 Roadmap: connect/split/merge}
Planned phases extend action space with explicit graph edits:
\begin{itemize}
  \item \textbf{Connect:} create/reweight candidate edges when repeated successful transitions lack direct structure.
  \item \textbf{Split:} divide overloaded nodes when policy confidence or error patterns indicate topic collapse.
  \item \textbf{Merge:} combine nodes with persistent redundancy and equivalent downstream behavior.
\end{itemize}
Each action class is expected to use conservative thresholds, rollback support, and offline shadow evaluation before broad enablement.

\section{4. Evaluation: Industry Baselines + Ablations}
Evaluation is organized around industry-standard baselines and fully reproducible artifacts. These results are mechanism proof for routing behavior under controlled harnesses; they are not, by themselves, full end-to-end product proof for a deployed OpenClaw runtime.

\subsection*{4.1 Baselines (Why These Comparisons)}
We compare against three industry baselines plus the learned router:
\begin{itemize}
  \item \textbf{Vector top-k} (\texttt{vector\_only}): plain embedding retrieval, no traversal.
  \item \textbf{Top-k + reranker} (\texttt{edge\_sim\_legacy}): deterministic re-ranking with \texttt{route\_mode=edge+sim}.
  \item \textbf{Pointer-chasing} (\texttt{edge\_sim\_legacy} with multi-hop traversal): iterative retrieval without learned routing.
  \item \textbf{OpenClawBrain learned} (\texttt{learned}) with ablation bounds \texttt{graph\_prior\_only} and \texttt{qtsim\_only}.
\end{itemize}
These modes are exposed in the evaluation harness in the main \texttt{openclawbrain} repo.

\subsection*{4.2 Core Metrics}
\begin{itemize}
  \item \textbf{Reward / accuracy}: task success or correctness rate (dataset dependent).
  \item \textbf{\% oracle gap closed}: fraction of the oracle gap closed over training epochs (expert-regions simulation).
  \item \textbf{Latency}: p50/p95 end-to-end query time from the eval harness.
  \item \textbf{Tool calls, tokens, cost}: measured in downstream agent runs; report as \texttt{TBD} if not available.
\end{itemize}

\subsection*{4.3 Figures (SVGs in \texttt{figures/eval})}
SVG sources live in \texttt{figures/eval/*.svg}; the PDF renders the PNG versions below.

\begin{figure}[t]
\centering
\includegraphics[width=0.95\textwidth]{figures/eval/reward_vs_epoch.png}
\vspace{0.4em}
\includegraphics[width=0.95\textwidth]{figures/eval/oracle_gap_closed.png}
\caption{Expert-regions simulation learning behavior: reward trajectory and fraction of oracle gap closed across training epochs.}
\label{fig:expert-regions-learning}
\end{figure}

\begin{figure}[t]
\centering
\includegraphics[width=0.9\textwidth]{figures/eval/ablation_bar_chart.png}
\caption{Ablation comparison across \texttt{vector\_only}, \texttt{graph\_prior\_only}, \texttt{qtsim\_only}, and learned routing.}
\label{fig:ablation-bar}
\end{figure}

\subsection*{4.4 Reporting Template (Do Not Invent Numbers)}
All unreproduced results must be marked \texttt{TBD} and tied to an output file path.

\begin{table}[h]
\centering
\small
\begin{tabular}{lcccccc}
\toprule
Setting & Reward/Acc & \% Oracle Gap & Latency p50/p95 & Tool Calls & Tokens & Cost \\
\midrule
Vector top-k (expert-regions sim) & 0.878629 / 0.8920 & TBD & TBD & TBD & TBD & TBD \\
Top-k + reranker & 0.879762 / 0.8935 & TBD & TBD & TBD & TBD & TBD \\
Pointer-chasing (graph prior only) & 0.555147 / 0.5290 & TBD & TBD & TBD & TBD & TBD \\
OpenClawBrain learned (mixed) & 0.935126 / 0.9680 & 96.74\% & TBD & TBD & TBD & TBD \\
\bottomrule
\end{tabular}
\caption{Expert-regions simulation baselines from reproducible runs. Each \texttt{TBD} must cite commit SHA, command, and artifact path (e.g., \texttt{/tmp/ocb\_eval.json}).}
\end{table}

\subsection*{4.5 Reproduction Workflow (TypeScript-First Package Surface)}
Commands (from \texttt{openclawbrain} repo) and expected outputs:
\begin{verbatim}
cd /path/to/openclawbrain
corepack enable
pnpm install
pnpm check
pnpm release:pack

pnpm run eval:workflow-proof -- --output-dir scratch/workflow-proof/latest
pnpm run eval:baseline -- --queries /path/to/queries.jsonl --output /tmp/ocb_eval.json
pnpm run eval:sim-expert-regions -- --output-dir /tmp/ocb_expert_regions
pnpm run eval:sim-two-cluster -- --output-dir /tmp/ocb_two_cluster
\end{verbatim}
Expected outputs include \texttt{/tmp/ocb\_eval.json}, \texttt{/tmp/ocb\_expert\_regions/simulation\_curve.csv}, and regenerated SVGs in \texttt{figures/eval/} using the plotting entrypoint documented in \texttt{docs/reproduce-eval.md}.

\section{5. Operational Notes}
OpenClawBrain now supports local embeddings out-of-the-box for operator-first deployment. This enables deterministic bootstrap and private/offline-friendly operation while keeping optional teacher workflows in asynchronous jobs.

\section{6. Conclusion}
The v12.2.6+ system is defined by four commitments: local hot-path retrieval, QTsim+graph-prior confidence mixing, distillation plus policy-gradient learning, and RL-native structure maintenance. Performance claims should be published only from reproducible evaluation runs with explicit provenance.

\end{document}
