fix robots

v3/src/pages/robots.txt.ts

@@ -1,26 +1,27 @@
 import type { APIRoute } from 'astro';
 
-import { AI_CRAWLER_POLICIES, getSiteOrigin } from '~/utils/seo';
+import { BLOCKED_CRAWLERS, SITEMAP_PATHS, getSiteOrigin } from '~/utils/seo';
 
 export const prerender = true;
 
 export const GET: APIRoute = ({ site }) => {
   const origin = getSiteOrigin(site);
   const lines = [
+    '# RustDesk - robots.txt',
+    `# ${origin}`,
+    '',
+    '# Every crawler is welcome, AI crawlers included. A bot with no group of its own',
+    '# falls back to this group, so new ones never need an entry here.',
     'User-agent: *',
     'Allow: /',
     'Disallow: /success',
     'Disallow: /cancel',
     '',
-    ...AI_CRAWLER_POLICIES.flatMap((policy) => [
-      `User-agent: ${policy.name}`,
-      `Allow: ${policy.allow}`,
-      'Disallow: /success',
-      'Disallow: /cancel',
-      '',
-    ]),
-    `Sitemap: ${origin}/sitemap-index.xml`,
+    '# Bulk scrapers that feed training corpora and send back no readers.',
+    ...BLOCKED_CRAWLERS.flatMap((name) => [`User-agent: ${name}`, 'Disallow: /', '']),
+    ...SITEMAP_PATHS.map((path) => `Sitemap: ${origin}${path}`),
     `Host: ${new URL(origin).host}`,
+    '',
   ];
 
   return new Response(lines.join('\n'), {

v3/src/utils/seo.ts

@@ -122,15 +122,12 @@ export const shouldIndexUtilityPage = (pathname: string) => {
   return !['success', 'cancel', 'utility'].includes(kind);
 };
 
-export const AI_CRAWLER_POLICIES = [
-  { name: 'GPTBot', allow: '/' },
-  { name: 'ChatGPT-User', allow: '/' },
-  { name: 'PerplexityBot', allow: '/' },
-  { name: 'ClaudeBot', allow: '/' },
-  { name: 'anthropic-ai', allow: '/' },
-  { name: 'Google-Extended', allow: '/' },
-  { name: 'Bingbot', allow: '/' },
-];
+// Crawlers are allowed by the `User-agent: *` group in robots.txt, so AI crawlers need no
+// entry of their own. Only bots we want to keep out are named here.
+export const BLOCKED_CRAWLERS = ['CCBot', 'img2dataset'];
+
+// The /docs section is a separate Hugo build, so its sitemap is not part of the Astro one.
+export const SITEMAP_PATHS = ['/sitemap-index.xml', '/docs/sitemap.xml'];
 
 export const getImportantSiteLinks = () => [
   { label: 'Homepage', url: `${SITE_URL}/` },

v3/vendor/integration/index.ts

@@ -1,12 +1,9 @@
-import fs from 'node:fs';
-import os from 'node:os';
-import type { AstroConfig, AstroIntegration } from 'astro';
+import type { AstroIntegration } from 'astro';
 
 import configBuilder, { type Config } from './utils/configBuilder';
 import loadConfig from './utils/loadConfig';
 
 export default ({ config: _themeConfig = 'src/config.yaml' } = {}): AstroIntegration => {
-  let cfg: AstroConfig;
   return {
     name: 'astrowind-integration',
 
@@ -68,49 +65,6 @@ export default ({ config: _themeConfig = 'src/config.yaml' } = {}): AstroIntegra
           buildLogger.info(`Astrowind config has been loaded.`);
         }
       },
-      'astro:config:done': async ({ config }) => {
-        cfg = config;
-      },
-
-      'astro:build:done': async ({ logger }) => {
-        const buildLogger = logger.fork('astrowind');
-        buildLogger.info('Updating `robots.txt` with `sitemap-index.xml` ...');
-
-        try {
-          const outDir = cfg.outDir;
-          const publicDir = cfg.publicDir;
-          const sitemapName = 'sitemap-index.xml';
-          const sitemapFile = new URL(sitemapName, outDir);
-          const robotsTxtFile = new URL('robots.txt', publicDir);
-          const robotsTxtFileInOut = new URL('robots.txt', outDir);
-
-          const hasIntegration =
-            Array.isArray(cfg?.integrations) &&
-            cfg.integrations?.find((e) => e?.name === '@astrojs/sitemap') !== undefined;
-          const sitemapExists = fs.existsSync(sitemapFile);
-
-          if (hasIntegration && sitemapExists) {
-            const robotsTxt = fs.readFileSync(robotsTxtFile, { encoding: 'utf8', flag: 'a+' });
-            const sitemapUrl = new URL(sitemapName, String(new URL(cfg.base, cfg.site)));
-            const pattern = /^Sitemap:(.*)$/m;
-
-            if (!pattern.test(robotsTxt)) {
-              fs.writeFileSync(robotsTxtFileInOut, `${robotsTxt}${os.EOL}${os.EOL}Sitemap: ${sitemapUrl}`, {
-                encoding: 'utf8',
-                flag: 'w',
-              });
-            } else {
-              fs.writeFileSync(robotsTxtFileInOut, robotsTxt.replace(pattern, `Sitemap: ${sitemapUrl}`), {
-                encoding: 'utf8',
-                flag: 'w',
-              });
-            }
-          }
-          // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        } catch (error) {
-          /* empty */
-        }
-      },
     },
   };
 };