fix sitemap

.github/workflows/deploy.yml

@@ -13,6 +13,8 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
+          # Hugo uses per-file Git history for accurate sitemap lastmod values.
+          fetch-depth: 0
           submodules: recursive
           persist-credentials: false
 

v3/astro.config.ts

@@ -14,10 +14,12 @@ import type { AstroIntegration } from 'astro';
 import astrowind from './vendor/integration';
 
 import { readingTimeRemarkPlugin, responsiveTablesRehypePlugin, lazyImagesRehypePlugin } from './src/utils/frontmatter';
+import { shouldIncludeInSitemap } from './src/utils/sitemap.mjs';
 
 import react from '@astrojs/react';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const locales = ['en', 'de', 'es', 'fr', 'it', 'ja', 'pt', 'zh-cn', 'zh-tw', 'ko', 'ar'];
 
 const hasExternalScripts = false;
 const whenExternalScripts = (items: (() => AstroIntegration) | (() => AstroIntegration)[] = []) =>
@@ -27,7 +29,7 @@ export default defineConfig({
   output: 'static',
   i18n: {
     defaultLocale: 'en',
-    locales: ['en', 'de', 'es', 'fr', 'it', 'ja', 'pt', 'zh-cn', 'zh-tw', 'ko', 'ar'],
+    locales,
     routing: {
       prefixDefaultLocale: true,
       redirectToDefaultLocale: false,
@@ -38,7 +40,9 @@ export default defineConfig({
     tailwind({
       applyBaseStyles: false,
     }),
-    sitemap(),
+    sitemap({
+      filter: (page) => shouldIncludeInSitemap(page, locales),
+    }),
     mdx(),
     icon({
       include: {

v3/src/utils/sitemap.mjs

@@ -0,0 +1,18 @@
+const NON_INDEXABLE_SECTIONS = new Set(['404', 'cancel', 'success', 'tag']);
+
+/**
+ * Keep sitemap entries aligned with the routes that are allowed to be indexed.
+ * Astro passes absolute page URLs to the sitemap filter.
+ */
+export const shouldIncludeInSitemap = (pageUrl, locales = []) => {
+  const segments = new URL(pageUrl).pathname.split('/').filter(Boolean);
+
+  if (locales.includes(segments[0])) {
+    segments.shift();
+  }
+
+  const blogPage = Number(segments[1]);
+  const isNoindexBlogPagination = segments[0] === 'blog' && /^\d+$/.test(segments[1] ?? '') && blogPage > 1;
+
+  return !NON_INDEXABLE_SECTIONS.has(segments[0]) && !isNoindexBlogPagination;
+};

v3/tests/sitemap.test.mjs

@@ -0,0 +1,30 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { shouldIncludeInSitemap } from '../src/utils/sitemap.mjs';
+
+const locales = ['en', 'de', 'es', 'fr', 'it', 'ja', 'pt', 'zh-cn', 'zh-tw', 'ko', 'ar'];
+
+test('excludes noindex utility routes from the sitemap', () => {
+  for (const path of ['/404', '/cancel', '/success', '/de/cancel', '/zh-cn/success']) {
+    assert.equal(shouldIncludeInSitemap(`https://rustdesk.com${path}`, locales), false, path);
+  }
+});
+
+test('excludes tag archives in every locale from the sitemap', () => {
+  for (const path of ['/tag/rustdesk', '/fr/tag/rustdesk', '/zh-tw/tag/rustdesk/2']) {
+    assert.equal(shouldIncludeInSitemap(`https://rustdesk.com${path}`, locales), false, path);
+  }
+});
+
+test('excludes noindex blog pagination in every locale', () => {
+  for (const path of ['/blog/2', '/de/blog/6', '/zh-cn/blog/3']) {
+    assert.equal(shouldIncludeInSitemap(`https://rustdesk.com${path}`, locales), false, path);
+  }
+});
+
+test('keeps indexable pages whose later path segments resemble excluded routes', () => {
+  for (const path of ['/', '/pricing', '/blog/tag', '/de/blog/success', '/category/tag']) {
+    assert.equal(shouldIncludeInSitemap(`https://rustdesk.com${path}`, locales), true, path);
+  }
+});