fix robots

v3/src/pages/robots.txt.ts

@@ -1,26 +1,27 @@
 import type { APIRoute } from 'astro';
 
-import { AI_CRAWLER_POLICIES, getSiteOrigin } from '~/utils/seo';
+import { BLOCKED_CRAWLERS, SITEMAP_PATHS, getSiteOrigin } from '~/utils/seo';
 
 export const prerender = true;
 
 export const GET: APIRoute = ({ site }) => {
   const origin = getSiteOrigin(site);
   const lines = [
+    '# RustDesk - robots.txt',
+    `# ${origin}`,
+    '',
+    '# Every crawler is welcome, AI crawlers included. A bot with no group of its own',
+    '# falls back to this group, so new ones never need an entry here.',
     'User-agent: *',
     'Allow: /',
     'Disallow: /success',
     'Disallow: /cancel',
     '',
-    ...AI_CRAWLER_POLICIES.flatMap((policy) => [
-      `User-agent: ${policy.name}`,
-      `Allow: ${policy.allow}`,
-      'Disallow: /success',
-      'Disallow: /cancel',
-      '',
-    ]),
-    `Sitemap: ${origin}/sitemap-index.xml`,
+    '# Bulk scrapers that feed training corpora and send back no readers.',
+    ...BLOCKED_CRAWLERS.flatMap((name) => [`User-agent: ${name}`, 'Disallow: /', '']),
+    ...SITEMAP_PATHS.map((path) => `Sitemap: ${origin}${path}`),
     `Host: ${new URL(origin).host}`,
+    '',
   ];
 
   return new Response(lines.join('\n'), {

v3/src/utils/seo.ts

@@ -122,15 +122,12 @@ export const shouldIndexUtilityPage = (pathname: string) => {
   return !['success', 'cancel', 'utility'].includes(kind);
 };
 
-export const AI_CRAWLER_POLICIES = [
-  { name: 'GPTBot', allow: '/' },
-  { name: 'ChatGPT-User', allow: '/' },
-  { name: 'PerplexityBot', allow: '/' },
-  { name: 'ClaudeBot', allow: '/' },
-  { name: 'anthropic-ai', allow: '/' },
-  { name: 'Google-Extended', allow: '/' },
-  { name: 'Bingbot', allow: '/' },
-];
+// Crawlers are allowed by the `User-agent: *` group in robots.txt, so AI crawlers need no
+// entry of their own. Only bots we want to keep out are named here.
+export const BLOCKED_CRAWLERS = ['CCBot', 'img2dataset'];
+
+// The /docs section is a separate Hugo build, so its sitemap is not part of the Astro one.
+export const SITEMAP_PATHS = ['/sitemap-index.xml', '/docs/sitemap.xml'];
 
 export const getImportantSiteLinks = () => [
   { label: 'Homepage', url: `${SITE_URL}/` },