fix(proxy): noindex llms.txt, llms-full.txt and changelog.xml on self-hosted deployments

Author: waleedlatif1

apps/sim/lib/landing/routes.ts

@@ -15,7 +15,8 @@
  *   landing footer's toggle, instead of the app's account-synced `system`
  *   default. Leave one out and a visitor on a dark OS lands on a dark
  *   marketing page.
- * - The proxy marks these paths `noindex` on self-hosted deployments, so a
+ * - The proxy marks these paths, and the marketing documents in
+ *   {@link isMarketingPath}, `noindex` on self-hosted deployments, so a
  *   third-party copy of the marketing site never competes with sim.ai in search
  *   or answer engines. Leave one out and that page stays indexable there.
  *
@@ -56,3 +57,19 @@ export function isLandingPath(pathname: string): boolean {
   const firstSegment = pathname.split('/')[1]
   return firstSegment === '' || LANDING_SEGMENTS.has(firstSegment)
 }
+
+/**
+ * Root-level documents that republish the marketing site's content for crawlers
+ * and answer engines. They are served outside `app/(landing)`, so
+ * {@link isLandingPath} does not cover them.
+ */
+const MARKETING_DOCUMENT_PATHS: ReadonlySet<string> = new Set([
+  '/llms.txt',
+  '/llms-full.txt',
+  '/changelog.xml',
+])
+
+/** Whether a pathname serves marketing content: a landing page or a marketing document. */
+export function isMarketingPath(pathname: string): boolean {
+  return isLandingPath(pathname) || MARKETING_DOCUMENT_PATHS.has(pathname)
+}

apps/sim/proxy.test.ts

@@ -173,10 +173,20 @@ describe('proxy indexing policy', () => {
     expect(robotsTag('/pricing')).toBe('noindex, nofollow')
   })
 
-  it('leaves the marketing pages of the hosted deployment indexable', () => {
+  it('keeps the marketing documents of a self-hosted deployment out of search results', () => {
+    setEnvFlags({ isHosted: false })
+    expect(robotsTag('/llms.txt')).toBe('noindex, nofollow')
+    expect(robotsTag('/llms-full.txt')).toBe('noindex, nofollow')
+    expect(robotsTag('/changelog.xml')).toBe('noindex, nofollow')
+  })
+
+  it('leaves the marketing pages and documents of the hosted deployment indexable', () => {
     setEnvFlags({ isHosted: true })
     expect(robotsTag('/comparisons/n8n')).toBeNull()
     expect(robotsTag('/pricing')).toBeNull()
+    expect(robotsTag('/llms.txt')).toBeNull()
+    expect(robotsTag('/llms-full.txt')).toBeNull()
+    expect(robotsTag('/changelog.xml')).toBeNull()
   })
 
   it('leaves a self-hosted non-landing public page indexable', () => {
@@ -199,6 +209,12 @@ describe('proxy matcher', () => {
     expect(matches('/api/internal/project-file-doc/project-1/file-1/persist')).toBe(false)
   })
 
+  it('runs for the marketing documents, so the indexing policy reaches them', () => {
+    expect(matches('/llms.txt')).toBe(true)
+    expect(matches('/llms-full.txt')).toBe(true)
+    expect(matches('/changelog.xml')).toBe(true)
+  })
+
   it('still runs for every other API route', () => {
     expect(matches('/api/desktop/tool/claim')).toBe(true)
     expect(matches('/api/desktop/tool/import/other')).toBe(true)

apps/sim/proxy.ts

@@ -7,7 +7,7 @@ import { getSessionCookie } from 'better-auth/cookies'
 import { type NextRequest, NextResponse } from 'next/server'
 import { resolveSimMcpHostPath } from '@/lib/api/mcp/host-routing'
 import { SIM_MCP_ROUTE_PATH } from '@/lib/api/mcp/urls'
-import { isLandingPath } from '@/lib/landing/routes'
+import { isMarketingPath } from '@/lib/landing/routes'
 import { APP_ENTRY_PATH, isAppSurfacePath, isNoindexPath } from '@/lib/navigation/paths'
 import { isOAuthAuthorizationCallback, resolveAuthRedirect } from '@/app/(auth)/auth-redirect'
 import { getEnv } from './lib/core/config/env'
@@ -457,7 +457,7 @@ function applyIndexingPolicy(request: NextRequest, response: NextResponse): Next
   if (
     isNonCanonicalSimHost(host) ||
     isNoindexPath(pathname) ||
-    (!isHosted && isLandingPath(pathname))
+    (!isHosted && isMarketingPath(pathname))
   ) {
     response.headers.set('X-Robots-Tag', 'noindex, nofollow')
   }