From f3bbb69b25d1b1619f7bd9d5861480af7e9e834f Mon Sep 17 00:00:00 2001
From: Agando <roduccomm@gmail.com>
Date: Fri, 23 Oct 2026 18:20:29 +0200
Subject: fix(newos4): tokenised body-aware skill search for load_skills
 (v4.6.62)

Packet 4. The matcher tested the whole query as one substring against
name+description+tags: word order decided the outcome ("telemetry dashboard" = 0
while "telemetry" = 1), a sentence could never match, and skill bodies were never
searched at all.

- _skillTokens(): lowercase, split on non-word chars, drop <3-char words and a
  stopword list; a query made only of stopwords keeps its literal form
- scoring: name 3, description 2, tags 2, body 1 + up to 1.0 for repeated hits,
  multiplied by token coverage (0.5 + 0.5 * matched/total), +3 phrase bonus so word
  order can never zero a real hit
- results carry score and matchedTokens ("2/3"); top 3 by score, name as tiebreak
- zero results now name the tokens searched and the closest skill names instead of
  "No matching skills found"

Verified live on v4.6.62 via the real runTool entry point:
"telemetry dashboard" and "dashboard telemetry" both return the same 2 skills
(was 0 and 0); "how do I regenerate the skill index" returns skills-system-guide
first (8.96, 2/3) where a sentence previously matched nothing; "info" returns 3
body matches; a nonsense query returns 0 with the tokens and closest names.
---
 help.html                       |  3 +-
 main.html                       |  6 +--
 modules/core/core-bundle.js     | 83 +++++++++++++++++++++++++++++++++++++----
 modules/core/module-aiassist.js | 83 +++++++++++++++++++++++++++++++++++++----
 modules/core/module-loader.js   |  2 +-
 5 files changed, 158 insertions(+), 19 deletions(-)

diff --git a/help.html b/help.html
index 60dbf4a..815727b 100644
--- a/help.html
+++ b/help.html
@@ -1672,12 +1672,13 @@ listmods</pre>
     <h2>📋 Changelog</h2>
 
     <div class="version-item">
-      <div class="v-header"><span class="v-num">v4.6.57–60 — October 2026</span><span class="v-date">2026-10-23</span></div>
+      <div class="v-header"><span class="v-num">v4.6.57–62 — October 2026</span><span class="v-date">2026-10-23</span></div>
       <ul>
         <li><strong>No shell — and it now says so</strong> — <code>;</code>, <code>&amp;&amp;</code> and <code>||</code> are refused with an explicit message instead of being folded into the command's argv (where <code>wc -l f &amp;&amp; grep -c x f</code> returned a plausible wrong number). A <code>|</code> arriving from the AI <code>run_command</code> tool is refused too, because that path bypasses the pipe splitter and degraded the pattern to empty. Documented at the top of the command reference.</li>
         <li><strong>Multiple paths in <code>wc</code></strong> — <code>wc -c a b</code> printed only <code>a</code> and exited clean. It now prints one line per file plus a <code>total</code>, and reports each missing file separately.</li>
         <li><strong><code>ls</code> prints text</strong> — it rendered only clickable chips (DOM-only), so any capture — the AI <code>run_command</code> tool, the diagnostics sweep — saw an empty result and reported "directory empty".</li>
         <li><strong><code>view_module</code> shows what is deployed</strong> — it returned the <code>_mods/&lt;Class&gt;.js</code> edit cache first, unversioned, so after a rebuild it showed pre-fix source while the bundle was new. It now fetches the deployed file and prefixes the version; the cache is a labelled fallback.</li>
+        <li><strong><code>load_skills</code> searches tokens, not one phrase</strong> — the query used to be tested whole against name+description+tags, so <code>"telemetry dashboard"</code> returned 0 while <code>"telemetry"</code> returned 1, and a sentence could never match. Skill <em>bodies</em> were never searched. It now tokenises (stopwords dropped), scores name &gt; description/tags &gt; body, weights by how many tokens matched, keeps a phrase bonus so word order cannot zero a real hit, and reports <code>matchedTokens: 2/3</code> with each result.</li>
         <li><strong>One error counter</strong> — <code>errors</code>, <code>get_dashboard_data</code> and the world model tallied three different sinks (0 / 18 / 11 in the same second). All three now read <code>terminal.countErrors()</code> (DebugSupervisor DB + failed tool calls); the VFS log is reported separately as a mirror, and <code>terminal.log('ERROR', …)</code> now records into the same DB instead of a second store nobody read.</li>
         <li><strong>Predictor arithmetic</strong> — the stored field named <code>mape</code> held a sum, so any JSON consumer read a mean that was <em>n</em> times too large (now <code>errorSum</code>); <code>errorPct</code> is stored raw and clamped only at render (a 475× miss used to be flattened to 500%); predictions are floored at 1 s and floor-hits are not scored, because one 0.079 s aborted run was rounding the next prediction to 0 s.</li>
       </ul>
diff --git a/main.html b/main.html
index ffac150..21f6eaa 100644
--- a/main.html
+++ b/main.html
@@ -1349,13 +1349,13 @@
     </script>
 
     <!-- v4.5.63: A2A scratchpad fixes + agent-register arg fix -->
-<script src="modules/core/core-bundle.js?v=v4.6.61"></script>
+<script src="modules/core/core-bundle.js?v=v4.6.62"></script>
 
     <!-- Async dependency-ordered module loader now only initializes terminal and loads optional ext modules -->
-<script src="modules/core/module-loader.js?v=v4.6.61"></script>
+<script src="modules/core/module-loader.js?v=v4.6.62"></script>
 
     <!-- Critical ext bundle loads AFTER webx11 exists so registerModule calls work -->
-<script src="modules/ext/critical-ext-bundle.js?v=v4.6.61"></script>
+<script src="modules/ext/critical-ext-bundle.js?v=v4.6.62"></script>
 
 <script>
 // ═══ ONE TRUE VERSION SOURCE ═══
diff --git a/modules/core/core-bundle.js b/modules/core/core-bundle.js
index e265b68..48d9dbe 100644
--- a/modules/core/core-bundle.js
+++ b/modules/core/core-bundle.js
@@ -11015,24 +11015,71 @@ buildSystemPrompt() {
                             return JSON.stringify({ message: `Agent spawned: ${agent.id}`, agent });
                         }
                         case 'load_skills': {
-                            const query = (a && a.query || '').toLowerCase();
+                            // v4.6.62: tokenised, body-aware search. The old matcher tested the
+                            // WHOLE query as one substring against name+description+tags, so word
+                            // order decided the outcome ("telemetry dashboard" = 0 while
+                            // "telemetry" = 1) and a sentence could never match anything. Skill
+                            // bodies were never searched at all (packet §4).
+                            const rawQuery = (a && a.query) || '';
+                            const query = String(rawQuery).toLowerCase().trim();
                             const fs = this.terminal.fs;
                             const index = fs.fsReadJSON('/skills/index.json') || {};
                             if (!query) {
                                 const names = Object.keys(index);
                                 return JSON.stringify({ skills: names, count: names.length, message: 'No query provided — showing all skill names. Call again with a query to load content.' });
                             }
+                            const tokens = this._skillTokens(query);
                             const matches = [];
                             for (const [name, meta] of Object.entries(index)) {
-                                const score = (name.includes(query) ? 3 : 0) + (meta.description.toLowerCase().includes(query) ? 2 : 0) + ((meta.tags || []).some(t => t.toLowerCase().includes(query)) ? 2 : 0);
-                                if (score > 0) {
-                                    const content = fs.readFile(`/skills/${name}.md`) || '';
-                                    matches.push({ name, description: meta.description, category: meta.category, score, content: content.substring(0, 2000) });
+                                const nameL = name.toLowerCase();
+                                const descL = String(meta.description || '').toLowerCase();
+                                const tagsL = (meta.tags || []).map(x => String(x).toLowerCase());
+                                const bodyRaw = fs.readFile('/skills/' + name + '.md') || '';
+                                const bodyL = bodyRaw.toLowerCase();
+                                let score = 0;
+                                let matched = 0;
+                                for (const tok of tokens) {
+                                    let hit = 0;
+                                    if (nameL.includes(tok)) hit += 3;
+                                    if (descL.includes(tok)) hit += 2;
+                                    if (tagsL.some(x => x.includes(tok))) hit += 2;
+                                    if (bodyL) {
+                                        const n = bodyL.split(tok).length - 1;
+                                        if (n > 0) hit += 1 + Math.min(n, 4) * 0.25;
+                                    }
+                                    if (hit > 0) { score += hit; matched++; }
                                 }
+                                if (!matched) continue;
+                                // phrase bonus: word order must never be able to produce a zero
+                                if ((nameL + ' ' + descL).includes(query)) score += 3;
+                                // coverage: matching 1 of 4 tokens is a weak hit, not a strong one
+                                score = score * (0.5 + 0.5 * (matched / tokens.length));
+                                matches.push({
+                                    name: name,
+                                    description: meta.description,
+                                    category: meta.category,
+                                    score: Math.round(score * 100) / 100,
+                                    matchedTokens: matched + '/' + tokens.length,
+                                    content: bodyRaw.substring(0, 2000)
+                                });
                             }
-                            matches.sort((a, b) => b.score - a.score);
+                            matches.sort((x, y) => y.score - x.score || (x.name < y.name ? -1 : 1));
                             const top = matches.slice(0, 3);
-                            return JSON.stringify(top.length > 0 ? { matches: top.map(m => ({ name: m.name, description: m.description, content: m.content })), count: top.length } : { matches: [], count: 0, message: 'No matching skills found. You can create one after this task.' });
+                            if (!top.length) {
+                                const names = Object.keys(index);
+                                return JSON.stringify({
+                                    matches: [], count: 0, tokens: tokens,
+                                    message: 'No skill matched any of: ' + tokens.join(', ') +
+                                        '. Closest skill names: ' + names.slice(0, 8).join(', ') +
+                                        '. You can create one after this task.'
+                                });
+                            }
+                            return JSON.stringify({
+                                matches: top.map(m => ({ name: m.name, description: m.description,
+                                    content: m.content, score: m.score, matchedTokens: m.matchedTokens })),
+                                count: top.length,
+                                tokens: tokens
+                            });
                         }
                         case 'create_skill': {
                             const name = (a.name || '').trim().replace(/\s+/g, '-').toLowerCase();
@@ -11403,6 +11450,28 @@ buildSystemPrompt() {
                     return JSON.stringify({ error: e.message, stack: e.stack?.substring(0, 200) });
                 }
             }
+            // v4.6.62: query tokeniser for load_skills — lowercase, split on non-word chars,
+            // drop short words and stopwords. A sentence query keeps its content tokens instead
+            // of collapsing to a phrase that can never match.
+            _skillTokens(query) {
+                const STOP = new Set(['the','and','for','with','how','what','when','which','who','why',
+                    'does','did','can','could','should','would','you','your','our','their','this','that',
+                    'these','those','from','into','about','over','under','use','using','used','get','got',
+                    'make','made','need','want','please','help','me','my','we','it','is','are','was','were',
+                    'be','been','to','of','in','on','at','by','or','not','no','do','i','a','an','as','if']);
+                const seen = new Set();
+                const out = [];
+                String(query || '').toLowerCase().split(/[^a-z0-9_\-]+/).forEach(w => {
+                    if (!w || w.length < 3 || STOP.has(w) || seen.has(w)) return;
+                    seen.add(w); out.push(w);
+                });
+                // A query made entirely of stopwords still deserves a literal attempt.
+                if (!out.length) {
+                    const literal = String(query || '').trim();
+                    if (literal) out.push(literal);
+                }
+                return out;
+            }
             async runModuleTool(name, argsStr) {
                 let a = {};
                 try {
diff --git a/modules/core/module-aiassist.js b/modules/core/module-aiassist.js
index 3326483..f5895f3 100644
--- a/modules/core/module-aiassist.js
+++ b/modules/core/module-aiassist.js
@@ -2698,24 +2698,71 @@ buildSystemPrompt() {
                             return JSON.stringify({ message: `Agent spawned: ${agent.id}`, agent });
                         }
                         case 'load_skills': {
-                            const query = (a && a.query || '').toLowerCase();
+                            // v4.6.62: tokenised, body-aware search. The old matcher tested the
+                            // WHOLE query as one substring against name+description+tags, so word
+                            // order decided the outcome ("telemetry dashboard" = 0 while
+                            // "telemetry" = 1) and a sentence could never match anything. Skill
+                            // bodies were never searched at all (packet §4).
+                            const rawQuery = (a && a.query) || '';
+                            const query = String(rawQuery).toLowerCase().trim();
                             const fs = this.terminal.fs;
                             const index = fs.fsReadJSON('/skills/index.json') || {};
                             if (!query) {
                                 const names = Object.keys(index);
                                 return JSON.stringify({ skills: names, count: names.length, message: 'No query provided — showing all skill names. Call again with a query to load content.' });
                             }
+                            const tokens = this._skillTokens(query);
                             const matches = [];
                             for (const [name, meta] of Object.entries(index)) {
-                                const score = (name.includes(query) ? 3 : 0) + (meta.description.toLowerCase().includes(query) ? 2 : 0) + ((meta.tags || []).some(t => t.toLowerCase().includes(query)) ? 2 : 0);
-                                if (score > 0) {
-                                    const content = fs.readFile(`/skills/${name}.md`) || '';
-                                    matches.push({ name, description: meta.description, category: meta.category, score, content: content.substring(0, 2000) });
+                                const nameL = name.toLowerCase();
+                                const descL = String(meta.description || '').toLowerCase();
+                                const tagsL = (meta.tags || []).map(x => String(x).toLowerCase());
+                                const bodyRaw = fs.readFile('/skills/' + name + '.md') || '';
+                                const bodyL = bodyRaw.toLowerCase();
+                                let score = 0;
+                                let matched = 0;
+                                for (const tok of tokens) {
+                                    let hit = 0;
+                                    if (nameL.includes(tok)) hit += 3;
+                                    if (descL.includes(tok)) hit += 2;
+                                    if (tagsL.some(x => x.includes(tok))) hit += 2;
+                                    if (bodyL) {
+                                        const n = bodyL.split(tok).length - 1;
+                                        if (n > 0) hit += 1 + Math.min(n, 4) * 0.25;
+                                    }
+                                    if (hit > 0) { score += hit; matched++; }
                                 }
+                                if (!matched) continue;
+                                // phrase bonus: word order must never be able to produce a zero
+                                if ((nameL + ' ' + descL).includes(query)) score += 3;
+                                // coverage: matching 1 of 4 tokens is a weak hit, not a strong one
+                                score = score * (0.5 + 0.5 * (matched / tokens.length));
+                                matches.push({
+                                    name: name,
+                                    description: meta.description,
+                                    category: meta.category,
+                                    score: Math.round(score * 100) / 100,
+                                    matchedTokens: matched + '/' + tokens.length,
+                                    content: bodyRaw.substring(0, 2000)
+                                });
                             }
-                            matches.sort((a, b) => b.score - a.score);
+                            matches.sort((x, y) => y.score - x.score || (x.name < y.name ? -1 : 1));
                             const top = matches.slice(0, 3);
-                            return JSON.stringify(top.length > 0 ? { matches: top.map(m => ({ name: m.name, description: m.description, content: m.content })), count: top.length } : { matches: [], count: 0, message: 'No matching skills found. You can create one after this task.' });
+                            if (!top.length) {
+                                const names = Object.keys(index);
+                                return JSON.stringify({
+                                    matches: [], count: 0, tokens: tokens,
+                                    message: 'No skill matched any of: ' + tokens.join(', ') +
+                                        '. Closest skill names: ' + names.slice(0, 8).join(', ') +
+                                        '. You can create one after this task.'
+                                });
+                            }
+                            return JSON.stringify({
+                                matches: top.map(m => ({ name: m.name, description: m.description,
+                                    content: m.content, score: m.score, matchedTokens: m.matchedTokens })),
+                                count: top.length,
+                                tokens: tokens
+                            });
                         }
                         case 'create_skill': {
                             const name = (a.name || '').trim().replace(/\s+/g, '-').toLowerCase();
@@ -3086,6 +3133,28 @@ buildSystemPrompt() {
                     return JSON.stringify({ error: e.message, stack: e.stack?.substring(0, 200) });
                 }
             }
+            // v4.6.62: query tokeniser for load_skills — lowercase, split on non-word chars,
+            // drop short words and stopwords. A sentence query keeps its content tokens instead
+            // of collapsing to a phrase that can never match.
+            _skillTokens(query) {
+                const STOP = new Set(['the','and','for','with','how','what','when','which','who','why',
+                    'does','did','can','could','should','would','you','your','our','their','this','that',
+                    'these','those','from','into','about','over','under','use','using','used','get','got',
+                    'make','made','need','want','please','help','me','my','we','it','is','are','was','were',
+                    'be','been','to','of','in','on','at','by','or','not','no','do','i','a','an','as','if']);
+                const seen = new Set();
+                const out = [];
+                String(query || '').toLowerCase().split(/[^a-z0-9_\-]+/).forEach(w => {
+                    if (!w || w.length < 3 || STOP.has(w) || seen.has(w)) return;
+                    seen.add(w); out.push(w);
+                });
+                // A query made entirely of stopwords still deserves a literal attempt.
+                if (!out.length) {
+                    const literal = String(query || '').trim();
+                    if (literal) out.push(literal);
+                }
+                return out;
+            }
             async runModuleTool(name, argsStr) {
                 let a = {};
                 try {
diff --git a/modules/core/module-loader.js b/modules/core/module-loader.js
index ddb2874..2871ef3 100644
--- a/modules/core/module-loader.js
+++ b/modules/core/module-loader.js
@@ -4,7 +4,7 @@
     // Core module bundle loaded synchronously in main.html.
     // This loader only handles version cache-busting, splash, terminal init, and optional ext modules.
 
-    const LATEST_VER = 'v4.6.61';
+    const LATEST_VER = 'v4.6.62';
     const CACHE = '?v=' + LATEST_VER;
     const stored = localStorage.getItem('webx11_cache_ver');
     if (stored !== LATEST_VER) {
-- 
cgit v1.2.3