diff --git a/docs/work/EXECUTE-seo-recovery.md b/docs/work/EXECUTE-seo-recovery.md index 23269a54..b078fd69 100644 --- a/docs/work/EXECUTE-seo-recovery.md +++ b/docs/work/EXECUTE-seo-recovery.md @@ -26,6 +26,24 @@ This brief supersedes the measurement plan in [EXECUTE-seo-keyword-rework.md](EX ## Log +- 2026-10-21 Follow-up verified the deployed reference-file repair from #662. + The huggingface `references/adapting-to-the-lora.md` link returned 200 HTML with its file title and noindex directive. + Deployment [39320257178](https://github.com/skilld-dev/skilld.dev/actions/runs/39320257178) succeeded for `c1e3ad7a`, which contains #661 and #662. + All 41 original panel URLs were checked. The 34 active URLs comprised 33 direct 200 responses and one canonical redirect. + The `taste-skill/taste-skill` URL redirected to `/gh/leonxlnx/taste-skill`, which returned 200 and allowed indexing. + Every active URL or its redirect target appeared in the current Sitemap. The Skills sitemap listed 127 URLs. + Preserve the original panel URL and its observations; the redirect does not transfer its history to the target. + All 41 watches retained the same latest Checkpoints as the 20 October read, spanning 14 to 18 October UTC. + The active set still reported 33 unknown and one crawled, not indexed. These are retained verdicts, not new inspections. + Search Console's available 11 to 17 October window reported zero clicks and 31 impressions; the previous window reported zero and 34. + The site assessment was provisional: its data quality was `degraded`, with two unavailable proofs. + Its ranked unknown-URL action remained count-only; its source snapshot was collected on 25 August. + The crawled, not indexed action listed five of 5,414 reported URLs from an October 1 sample, including retired URLs. + Neither action identifies a new sitewide repair cause. Keep intentional exclusions and the 11 November gate. + Live Markdown twins for an excluded Skill and a multi-Skill Repository hub returned an index header. + Their HTML pages returned noindex. The repair applies the existing Skill admission rule to direct Markdown sources. + HTML conversion now carries the rendered robots directive into the Markdown response. + - 2026-10-20 NuxtSEO access succeeded after renewed browser pairing. CLI 0.5.10 read all 41 panel watches. Every watch had a Checkpoint. Latest Checkpoints span `2026-10-14T21:09:21Z` to `2026-10-18T08:46:05Z`. All predate Monday 19 October in Melbourne. This read cannot complete that Monday's comparison. diff --git a/server/plugins/ai-ready-markdown-source.ts b/server/plugins/ai-ready-markdown-source.ts index 0fb958cd..18d8bf06 100644 --- a/server/plugins/ai-ready-markdown-source.ts +++ b/server/plugins/ai-ready-markdown-source.ts @@ -1,8 +1,10 @@ -import { getQuery } from 'h3' +import type { SkillIndexInput } from '#layers/registry/server/utils/trending-admission' +import { getQuery, setResponseHeader } from 'h3' import { composeAgentSkillMarkdown, composeSkillForkMarkdown } from '#layers/registry/server/utils/skill-agent-markdown' +import { isSkillIndexable, SKILL_INDEX_INPUT_COLUMNS_SQL } from '#layers/registry/server/utils/trending-admission' import { repositoryIdentityFromGhPath, skillIdentityFromGhPath } from '#shared/skill-markdown-route' -interface RenderedSkillRow { +interface RenderedSkillRow extends SkillIndexInput { name: string rendered_status: string | null rendered_raw: string | null @@ -41,10 +43,11 @@ export default defineNitroPlugin((nitroApp) => { // Match the resolved count that chooses the Skill page's canonical URL. // Two rows prove ambiguity. Do not pick an arbitrary Skill from a hub. const rows = await db - .prepare(`SELECT name, rendered_status, rendered_raw, display_name, description, rendered_at, assets - FROM skills - WHERE owner = ? AND repo = ? AND source_resolved = 1 - ${identity ? 'AND name = ?' : ''} + .prepare(`SELECT s.name, s.rendered_status, s.rendered_raw, s.display_name, s.description, s.rendered_at, s.assets, + s.seo_indexable, ${SKILL_INDEX_INPUT_COLUMNS_SQL} + FROM skills s JOIN repos r ON r.owner = s.owner AND r.repo = s.repo + WHERE s.owner = ? AND s.repo = ? AND s.source_resolved = 1 + ${identity ? 'AND s.name = ?' : ''} LIMIT 2`) .bind(...(identity ? [target.owner, target.repo, identity.name] : [target.owner, target.repo])) .all() @@ -63,6 +66,9 @@ export default defineNitroPlugin((nitroApp) => { if (row.rendered_status !== 'ok' || !row.rendered_raw) return + // The source shortcut skips HTML, so repeat the same admission rule here. + setResponseHeader(context.event, 'x-robots-tag', isSkillIndexable(row) ? 'index,follow' : 'noindex,follow') + context.source = { markdown: getQuery(context.event).action === 'fork' ? composeSkillForkMarkdown({ ...target, name: row.name }) diff --git a/server/plugins/robots-header.ts b/server/plugins/robots-header.ts index 2ed744eb..cffce16b 100644 --- a/server/plugins/robots-header.ts +++ b/server/plugins/robots-header.ts @@ -8,6 +8,14 @@ import { robotsFromHtml } from '#shared/robots-meta' * A response with no robots meta tag keeps whatever header it already has. */ export default defineNitroPlugin((nitroApp) => { + nitroApp.hooks.hook('ai-ready:page:markdown', ({ html, event }) => { + if (!event) + return + const directive = robotsFromHtml(html) + if (directive) + setResponseHeader(event, 'x-robots-tag', directive) + }) + nitroApp.hooks.hook('render:html', (html, { event }) => { const directive = robotsFromHtml(html.head.join('')) if (directive) diff --git a/test/unit/agent-skill-markdown-source.test.ts b/test/unit/agent-skill-markdown-source.test.ts index 127f2f79..c5fcf6c2 100644 --- a/test/unit/agent-skill-markdown-source.test.ts +++ b/test/unit/agent-skill-markdown-source.test.ts @@ -1,4 +1,5 @@ import Database from 'better-sqlite3' +import { createApp, defineEventHandler, toWebHandler } from 'h3' import { afterEach, describe, expect, it, vi } from 'vitest' import registerPlugin from '../../server/plugins/ai-ready-markdown-source' @@ -18,16 +19,22 @@ afterEach(() => { db.close() }) -async function requestMarkdown(route: string, skills: { name: string, resolved?: number, status?: string }[], action?: string) { +async function requestMarkdownResponse(route: string, skills: { name: string, resolved?: number, status?: string, indexable?: number, admitted?: boolean, probe?: boolean }[], action?: string, repoKind = 'creator') { const db = new Database(':memory:') databases.push(db) db.exec(`CREATE TABLE skills ( owner TEXT, repo TEXT, name TEXT, rendered_raw TEXT, display_name TEXT, - description TEXT, rendered_at INTEGER, assets TEXT, rendered_status TEXT, source_resolved INTEGER - )`) + description TEXT, rendered_at INTEGER, assets TEXT, rendered_status TEXT, source_resolved INTEGER, + seo_indexable INTEGER + ); + CREATE TABLE repos (owner TEXT, repo TEXT, repo_kind TEXT); + CREATE TABLE skill_trending_admissions (owner TEXT, repo TEXT, name TEXT, first_board TEXT);`) + db.prepare('INSERT INTO repos VALUES (?, ?, ?)').run('author', 'repository', repoKind) for (const skill of skills) { - db.prepare('INSERT INTO skills VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)') - .run('author', 'repository', skill.name, `---\nname: ${skill.name}\n---\n\nOriginal source.`, null, null, 1, '[]', skill.status ?? 'ok', skill.resolved ?? 1) + db.prepare('INSERT INTO skills VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)') + .run('author', 'repository', skill.name, `---\nname: ${skill.name}\n---\n\nOriginal source.`, null, null, 1, '[]', skill.status ?? 'ok', skill.resolved ?? 1, skill.indexable ?? 1) + if (skill.admitted || skill.probe) + db.prepare('INSERT INTO skill_trending_admissions VALUES (?, ?, ?, ?)').run('author', 'repository', skill.name, skill.probe ? 'probe' : 'week') } const binding = { prepare(sql: string) { @@ -42,22 +49,46 @@ async function requestMarkdown(route: string, skills: { name: string, resolved?: }, } let handler: ((context: MarkdownContext) => Promise) | undefined - const app = { + const nitroApp = { hooks: { hook(_name: string, callback: typeof handler) { handler = callback }, }, } - registerPlugin(app as unknown as Parameters[0]) + registerPlugin(nitroApp as unknown as Parameters[0]) if (!handler) throw new Error('The Markdown source hook was not registered') - const context: MarkdownContext = { route, event: { path: `${route}.md${action ? `?action=${action}` : ''}`, context: { platform: { db: binding } } } } - await handler(context) - return context.source + const sourceHandler = handler + const app = createApp() + app.use(defineEventHandler(async (event) => { + event.context.platform = { db: binding } + const context = { route, event, source: undefined } + await sourceHandler(context) + return { source: context.source } + })) + return toWebHandler(app)(new Request(`http://localhost${route}.md${action ? `?action=${action}` : ''}`)) +} + +async function requestMarkdown(...args: Parameters) { + const response = await requestMarkdownResponse(...args) + return (await response.json()).source as MarkdownContext['source'] } describe('agent Skill page Markdown source', () => { + it.each([ + { label: 'quality exclusion', skills: [{ name: 'writing', indexable: 0 }], repoKind: 'creator', robots: 'noindex,follow' }, + { label: 'unadmitted Skill', skills: [{ name: 'writing' }, { name: 'other', admitted: true }], repoKind: 'creator', robots: 'noindex,follow' }, + { label: 'admitted Skill', skills: [{ name: 'writing', admitted: true }], repoKind: 'creator', robots: 'index,follow' }, + { label: 'probe exception', skills: [{ name: 'writing', indexable: 0, probe: true }], repoKind: 'creator', robots: 'index,follow' }, + { label: 'empty admission fallback', skills: [{ name: 'writing' }], repoKind: 'creator', robots: 'index,follow' }, + { label: 'aggregator', skills: [{ name: 'writing', admitted: true }], repoKind: 'aggregator', robots: 'noindex,follow' }, + ])('serves the existing indexing rule for $label in Markdown', async ({ skills, repoKind, robots }) => { + const response = await requestMarkdownResponse('/gh/author/repository/writing', skills, undefined, repoKind) + expect(response.headers.get('x-robots-tag')).toBe(robots) + expect((await response.json()).source.markdown).toContain('Original source.') + }) + it.each(['/gh/author/repository', '/gh/author/repository/writing'])('serves only the fork workflow for %s?action=fork', async (route) => { const source = await requestMarkdown(route, [{ name: 'writing' }], 'fork') diff --git a/test/unit/markdown-robots.test.ts b/test/unit/markdown-robots.test.ts new file mode 100644 index 00000000..325fd71e --- /dev/null +++ b/test/unit/markdown-robots.test.ts @@ -0,0 +1,40 @@ +import type { H3Event } from 'h3' +import { createApp, defineEventHandler, toWebHandler } from 'h3' +import { describe, expect, it, vi } from 'vitest' +import registerPlugin from '../../server/plugins/robots-header' + +vi.hoisted(() => { + vi.stubGlobal('defineNitroPlugin', (plugin: unknown) => plugin) +}) + +async function convert(html: string, initialRobots = 'index,follow') { + let onMarkdown: ((context: { html: string, event?: H3Event }) => void) | undefined + registerPlugin({ hooks: { hook(name: string, callback: typeof onMarkdown) { + if (name === 'ai-ready:page:markdown') + onMarkdown = callback + } } } as unknown as Parameters[0]) + const app = createApp() + app.use(defineEventHandler((event) => { + event.node.res.setHeader('x-robots-tag', initialRobots) + onMarkdown?.({ html, event }) + return '# Converted page' + })) + return toWebHandler(app)(new Request('http://localhost/gh/author/repository.md')) +} + +describe('converted Markdown indexing directives', () => { + it('keeps an excluded HTML page excluded after conversion', async () => { + const response = await convert('

Repository

') + expect(response.headers.get('x-robots-tag')).toBe('noindex,follow') + }) + + it('retains the index directive from an admitted HTML page', async () => { + const response = await convert('

Skill

', 'noindex,follow') + expect(response.headers.get('x-robots-tag')).toBe('index,follow') + }) + + it('preserves the response policy when the HTML has no directive', async () => { + const response = await convert('

Page

', 'noindex,follow') + expect(response.headers.get('x-robots-tag')).toBe('noindex,follow') + }) +})