diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 1e1456a3..5bdbf46e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,6 +13,8 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: + # Hugo uses per-file Git history for accurate sitemap lastmod values. + fetch-depth: 0 submodules: recursive persist-credentials: false diff --git a/v3/astro.config.ts b/v3/astro.config.ts index 4beba687..c7117bf2 100644 --- a/v3/astro.config.ts +++ b/v3/astro.config.ts @@ -14,10 +14,12 @@ import type { AstroIntegration } from 'astro'; import astrowind from './vendor/integration'; import { readingTimeRemarkPlugin, responsiveTablesRehypePlugin, lazyImagesRehypePlugin } from './src/utils/frontmatter'; +import { shouldIncludeInSitemap } from './src/utils/sitemap.mjs'; import react from '@astrojs/react'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const locales = ['en', 'de', 'es', 'fr', 'it', 'ja', 'pt', 'zh-cn', 'zh-tw', 'ko', 'ar']; const hasExternalScripts = false; const whenExternalScripts = (items: (() => AstroIntegration) | (() => AstroIntegration)[] = []) => @@ -27,7 +29,7 @@ export default defineConfig({ output: 'static', i18n: { defaultLocale: 'en', - locales: ['en', 'de', 'es', 'fr', 'it', 'ja', 'pt', 'zh-cn', 'zh-tw', 'ko', 'ar'], + locales, routing: { prefixDefaultLocale: true, redirectToDefaultLocale: false, @@ -38,7 +40,9 @@ export default defineConfig({ tailwind({ applyBaseStyles: false, }), - sitemap(), + sitemap({ + filter: (page) => shouldIncludeInSitemap(page, locales), + }), mdx(), icon({ include: { diff --git a/v3/src/utils/sitemap.mjs b/v3/src/utils/sitemap.mjs new file mode 100644 index 00000000..00fb70bc --- /dev/null +++ b/v3/src/utils/sitemap.mjs @@ -0,0 +1,18 @@ +const NON_INDEXABLE_SECTIONS = new Set(['404', 'cancel', 'success', 'tag']); + +/** + * Keep sitemap entries aligned with the routes that are allowed to be indexed. + * Astro passes absolute page URLs to the sitemap filter. + */ +export const shouldIncludeInSitemap = (pageUrl, locales = []) => { + const segments = new URL(pageUrl).pathname.split('/').filter(Boolean); + + if (locales.includes(segments[0])) { + segments.shift(); + } + + const blogPage = Number(segments[1]); + const isNoindexBlogPagination = segments[0] === 'blog' && /^\d+$/.test(segments[1] ?? '') && blogPage > 1; + + return !NON_INDEXABLE_SECTIONS.has(segments[0]) && !isNoindexBlogPagination; +}; diff --git a/v3/tests/sitemap.test.mjs b/v3/tests/sitemap.test.mjs new file mode 100644 index 00000000..1431f1a0 --- /dev/null +++ b/v3/tests/sitemap.test.mjs @@ -0,0 +1,30 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { shouldIncludeInSitemap } from '../src/utils/sitemap.mjs'; + +const locales = ['en', 'de', 'es', 'fr', 'it', 'ja', 'pt', 'zh-cn', 'zh-tw', 'ko', 'ar']; + +test('excludes noindex utility routes from the sitemap', () => { + for (const path of ['/404', '/cancel', '/success', '/de/cancel', '/zh-cn/success']) { + assert.equal(shouldIncludeInSitemap(`https://rustdesk.com${path}`, locales), false, path); + } +}); + +test('excludes tag archives in every locale from the sitemap', () => { + for (const path of ['/tag/rustdesk', '/fr/tag/rustdesk', '/zh-tw/tag/rustdesk/2']) { + assert.equal(shouldIncludeInSitemap(`https://rustdesk.com${path}`, locales), false, path); + } +}); + +test('excludes noindex blog pagination in every locale', () => { + for (const path of ['/blog/2', '/de/blog/6', '/zh-cn/blog/3']) { + assert.equal(shouldIncludeInSitemap(`https://rustdesk.com${path}`, locales), false, path); + } +}); + +test('keeps indexable pages whose later path segments resemble excluded routes', () => { + for (const path of ['/', '/pricing', '/blog/tag', '/de/blog/success', '/category/tag']) { + assert.equal(shouldIncludeInSitemap(`https://rustdesk.com${path}`, locales), true, path); + } +});