fix(seo): preserve indexing rules in Markdown responses

docs/work/EXECUTE-seo-recovery.md

@@ -26,6 +26,24 @@ This brief supersedes the measurement plan in [EXECUTE-seo-keyword-rework.md](EX
 
 ## Log
 
+- 2026-10-21 Follow-up verified the deployed reference-file repair from #662.
+  The huggingface `references/adapting-to-the-lora.md` link returned 200 HTML with its file title and noindex directive.
+  Deployment [39320257178](https://github.com/skilld-dev/skilld.dev/actions/runs/39320257178) succeeded for `c1e3ad7a`, which contains #661 and #662.
+  All 41 original panel URLs were checked. The 34 active URLs comprised 33 direct 200 responses and one canonical redirect.
+  The `taste-skill/taste-skill` URL redirected to `/gh/leonxlnx/taste-skill`, which returned 200 and allowed indexing.
+  Every active URL or its redirect target appeared in the current Sitemap. The Skills sitemap listed 127 URLs.
+  Preserve the original panel URL and its observations; the redirect does not transfer its history to the target.
+  All 41 watches retained the same latest Checkpoints as the 20 October read, spanning 14 to 18 October UTC.
+  The active set still reported 33 unknown and one crawled, not indexed. These are retained verdicts, not new inspections.
+  Search Console's available 11 to 17 October window reported zero clicks and 31 impressions; the previous window reported zero and 34.
+  The site assessment was provisional: its data quality was `degraded`, with two unavailable proofs.
+  Its ranked unknown-URL action remained count-only; its source snapshot was collected on 25 August.
+  The crawled, not indexed action listed five of 5,414 reported URLs from an October 1 sample, including retired URLs.
+  Neither action identifies a new sitewide repair cause. Keep intentional exclusions and the 11 November gate.
+  Live Markdown twins for an excluded Skill and a multi-Skill Repository hub returned an index header.
+  Their HTML pages returned noindex. The repair applies the existing Skill admission rule to direct Markdown sources.
+  HTML conversion now carries the rendered robots directive into the Markdown response.
+
 - 2026-10-20 NuxtSEO access succeeded after renewed browser pairing. CLI 0.5.10 read all 41 panel watches.
   Every watch had a Checkpoint. Latest Checkpoints span `2026-10-14T21:09:21Z` to `2026-10-18T08:46:05Z`.
   All predate Monday 19 October in Melbourne. This read cannot complete that Monday's comparison.

server/plugins/ai-ready-markdown-source.ts

@@ -1,8 +1,10 @@
-import { getQuery } from 'h3'
+import type { SkillIndexInput } from '#layers/registry/server/utils/trending-admission'
+import { getQuery, setResponseHeader } from 'h3'
 import { composeAgentSkillMarkdown, composeSkillForkMarkdown } from '#layers/registry/server/utils/skill-agent-markdown'
+import { isSkillIndexable, SKILL_INDEX_INPUT_COLUMNS_SQL } from '#layers/registry/server/utils/trending-admission'
 import { repositoryIdentityFromGhPath, skillIdentityFromGhPath } from '#shared/skill-markdown-route'
 
-interface RenderedSkillRow {
+interface RenderedSkillRow extends SkillIndexInput {
   name: string
   rendered_status: string | null
   rendered_raw: string | null
@@ -41,10 +43,11 @@ export default defineNitroPlugin((nitroApp) => {
     // Match the resolved count that chooses the Skill page's canonical URL.
     // Two rows prove ambiguity. Do not pick an arbitrary Skill from a hub.
     const rows = await db
-      .prepare(`SELECT name, rendered_status, rendered_raw, display_name, description, rendered_at, assets
-                  FROM skills
-                 WHERE owner = ? AND repo = ? AND source_resolved = 1
-                   ${identity ? 'AND name = ?' : ''}
+      .prepare(`SELECT s.name, s.rendered_status, s.rendered_raw, s.display_name, s.description, s.rendered_at, s.assets,
+                       s.seo_indexable, ${SKILL_INDEX_INPUT_COLUMNS_SQL}
+                  FROM skills s JOIN repos r ON r.owner = s.owner AND r.repo = s.repo
+                 WHERE s.owner = ? AND s.repo = ? AND s.source_resolved = 1
+                   ${identity ? 'AND s.name = ?' : ''}
                  LIMIT 2`)
       .bind(...(identity ? [target.owner, target.repo, identity.name] : [target.owner, target.repo]))
       .all<RenderedSkillRow>()
@@ -63,6 +66,9 @@ export default defineNitroPlugin((nitroApp) => {
     if (row.rendered_status !== 'ok' || !row.rendered_raw)
       return
 
+    // The source shortcut skips HTML, so repeat the same admission rule here.
+    setResponseHeader(context.event, 'x-robots-tag', isSkillIndexable(row) ? 'index,follow' : 'noindex,follow')
+
     context.source = {
       markdown: getQuery(context.event).action === 'fork'
         ? composeSkillForkMarkdown({ ...target, name: row.name })

server/plugins/robots-header.ts

@@ -8,6 +8,14 @@ import { robotsFromHtml } from '#shared/robots-meta'
  * A response with no robots meta tag keeps whatever header it already has.
  */
 export default defineNitroPlugin((nitroApp) => {
+  nitroApp.hooks.hook('ai-ready:page:markdown', ({ html, event }) => {
+    if (!event)
+      return
+    const directive = robotsFromHtml(html)
+    if (directive)
+      setResponseHeader(event, 'x-robots-tag', directive)
+  })
+
   nitroApp.hooks.hook('render:html', (html, { event }) => {
     const directive = robotsFromHtml(html.head.join(''))
     if (directive)

test/unit/agent-skill-markdown-source.test.ts

@@ -1,4 +1,5 @@
 import Database from 'better-sqlite3'
+import { createApp, defineEventHandler, toWebHandler } from 'h3'
 import { afterEach, describe, expect, it, vi } from 'vitest'
 import registerPlugin from '../../server/plugins/ai-ready-markdown-source'
 
@@ -18,16 +19,22 @@ afterEach(() => {
     db.close()
 })
 
-async function requestMarkdown(route: string, skills: { name: string, resolved?: number, status?: string }[], action?: string) {
+async function requestMarkdownResponse(route: string, skills: { name: string, resolved?: number, status?: string, indexable?: number, admitted?: boolean, probe?: boolean }[], action?: string, repoKind = 'creator') {
   const db = new Database(':memory:')
   databases.push(db)
   db.exec(`CREATE TABLE skills (
     owner TEXT, repo TEXT, name TEXT, rendered_raw TEXT, display_name TEXT,
-    description TEXT, rendered_at INTEGER, assets TEXT, rendered_status TEXT, source_resolved INTEGER
-  )`)
+    description TEXT, rendered_at INTEGER, assets TEXT, rendered_status TEXT, source_resolved INTEGER,
+    seo_indexable INTEGER
+  );
+  CREATE TABLE repos (owner TEXT, repo TEXT, repo_kind TEXT);
+  CREATE TABLE skill_trending_admissions (owner TEXT, repo TEXT, name TEXT, first_board TEXT);`)
+  db.prepare('INSERT INTO repos VALUES (?, ?, ?)').run('author', 'repository', repoKind)
   for (const skill of skills) {
-    db.prepare('INSERT INTO skills VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
-      .run('author', 'repository', skill.name, `---\nname: ${skill.name}\n---\n\nOriginal source.`, null, null, 1, '[]', skill.status ?? 'ok', skill.resolved ?? 1)
+    db.prepare('INSERT INTO skills VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
+      .run('author', 'repository', skill.name, `---\nname: ${skill.name}\n---\n\nOriginal source.`, null, null, 1, '[]', skill.status ?? 'ok', skill.resolved ?? 1, skill.indexable ?? 1)
+    if (skill.admitted || skill.probe)
+      db.prepare('INSERT INTO skill_trending_admissions VALUES (?, ?, ?, ?)').run('author', 'repository', skill.name, skill.probe ? 'probe' : 'week')
   }
   const binding = {
     prepare(sql: string) {
@@ -42,22 +49,46 @@ async function requestMarkdown(route: string, skills: { name: string, resolved?:
     },
   }
   let handler: ((context: MarkdownContext) => Promise<void>) | undefined
-  const app = {
+  const nitroApp = {
     hooks: {
       hook(_name: string, callback: typeof handler) {
         handler = callback
       },
     },
   }
-  registerPlugin(app as unknown as Parameters<typeof registerPlugin>[0])
+  registerPlugin(nitroApp as unknown as Parameters<typeof registerPlugin>[0])
   if (!handler)
     throw new Error('The Markdown source hook was not registered')
-  const context: MarkdownContext = { route, event: { path: `${route}.md${action ? `?action=${action}` : ''}`, context: { platform: { db: binding } } } }
-  await handler(context)
-  return context.source
+  const sourceHandler = handler
+  const app = createApp()
+  app.use(defineEventHandler(async (event) => {
+    event.context.platform = { db: binding }
+    const context = { route, event, source: undefined }
+    await sourceHandler(context)
+    return { source: context.source }
+  }))
+  return toWebHandler(app)(new Request(`http://localhost${route}.md${action ? `?action=${action}` : ''}`))
+}
+
+async function requestMarkdown(...args: Parameters<typeof requestMarkdownResponse>) {
+  const response = await requestMarkdownResponse(...args)
+  return (await response.json()).source as MarkdownContext['source']
 }
 
 describe('agent Skill page Markdown source', () => {
+  it.each([
+    { label: 'quality exclusion', skills: [{ name: 'writing', indexable: 0 }], repoKind: 'creator', robots: 'noindex,follow' },
+    { label: 'unadmitted Skill', skills: [{ name: 'writing' }, { name: 'other', admitted: true }], repoKind: 'creator', robots: 'noindex,follow' },
+    { label: 'admitted Skill', skills: [{ name: 'writing', admitted: true }], repoKind: 'creator', robots: 'index,follow' },
+    { label: 'probe exception', skills: [{ name: 'writing', indexable: 0, probe: true }], repoKind: 'creator', robots: 'index,follow' },
+    { label: 'empty admission fallback', skills: [{ name: 'writing' }], repoKind: 'creator', robots: 'index,follow' },
+    { label: 'aggregator', skills: [{ name: 'writing', admitted: true }], repoKind: 'aggregator', robots: 'noindex,follow' },
+  ])('serves the existing indexing rule for $label in Markdown', async ({ skills, repoKind, robots }) => {
+    const response = await requestMarkdownResponse('/gh/author/repository/writing', skills, undefined, repoKind)
+    expect(response.headers.get('x-robots-tag')).toBe(robots)
+    expect((await response.json()).source.markdown).toContain('Original source.')
+  })
+
   it.each(['/gh/author/repository', '/gh/author/repository/writing'])('serves only the fork workflow for %s?action=fork', async (route) => {
     const source = await requestMarkdown(route, [{ name: 'writing' }], 'fork')
 

test/unit/markdown-robots.test.ts

@@ -0,0 +1,40 @@
+import type { H3Event } from 'h3'
+import { createApp, defineEventHandler, toWebHandler } from 'h3'
+import { describe, expect, it, vi } from 'vitest'
+import registerPlugin from '../../server/plugins/robots-header'
+
+vi.hoisted(() => {
+  vi.stubGlobal('defineNitroPlugin', (plugin: unknown) => plugin)
+})
+
+async function convert(html: string, initialRobots = 'index,follow') {
+  let onMarkdown: ((context: { html: string, event?: H3Event }) => void) | undefined
+  registerPlugin({ hooks: { hook(name: string, callback: typeof onMarkdown) {
+    if (name === 'ai-ready:page:markdown')
+      onMarkdown = callback
+  } } } as unknown as Parameters<typeof registerPlugin>[0])
+  const app = createApp()
+  app.use(defineEventHandler((event) => {
+    event.node.res.setHeader('x-robots-tag', initialRobots)
+    onMarkdown?.({ html, event })
+    return '# Converted page'
+  }))
+  return toWebHandler(app)(new Request('http://localhost/gh/author/repository.md'))
+}
+
+describe('converted Markdown indexing directives', () => {
+  it('keeps an excluded HTML page excluded after conversion', async () => {
+    const response = await convert('<meta name="robots" content="noindex,follow"><h1>Repository</h1>')
+    expect(response.headers.get('x-robots-tag')).toBe('noindex,follow')
+  })
+
+  it('retains the index directive from an admitted HTML page', async () => {
+    const response = await convert('<meta name="robots" content="index,follow"><h1>Skill</h1>', 'noindex,follow')
+    expect(response.headers.get('x-robots-tag')).toBe('index,follow')
+  })
+
+  it('preserves the response policy when the HTML has no directive', async () => {
+    const response = await convert('<h1>Page</h1>', 'noindex,follow')
+    expect(response.headers.get('x-robots-tag')).toBe('noindex,follow')
+  })
+})