Skip to content

Commit 614156b

Browse files
satya164satya164
committed
feat: automated generation of llms.txt for all versions
1 parent d286f54 commit 614156b

2 files changed

Lines changed: 230 additions & 2 deletions

File tree

‎docusaurus.config.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import remarkRawMarkdown from './src/plugins/remark-raw-markdown.mjs';
99
import darkTheme from './src/themes/react-navigation-dark';
1010
import lightTheme from './src/themes/react-navigation-light';
1111

12+
const latestVersion = '7.x';
13+
1214
const config: Config = {
1315
title: 'React Navigation',
1416
tagline: 'Routing and navigation for your React Native apps',
@@ -131,6 +133,10 @@ const config: Config = {
131133
plugins: [
132134
'./src/plugins/disable-fully-specified.mjs',
133135
'./src/plugins/react-navigation-versions.mjs',
136+
[
137+
'./src/plugins/generate-llms-txt.mjs',
138+
{ latestVersion, baseUrl: 'https://reactnavigation.org' },
139+
],
134140
[
135141
'@docusaurus/plugin-client-redirects',
136142
{
@@ -159,9 +165,9 @@ const config: Config = {
159165
editUrl:
160166
'https://github.com/react-navigation/react-navigation.github.io/edit/main/',
161167
includeCurrentVersion: false,
162-
lastVersion: '7.x',
168+
lastVersion: latestVersion,
163169
versions: {
164-
'7.x': {
170+
[latestVersion]: {
165171
badge: false,
166172
},
167173
},

‎src/plugins/generate-llms-txt.mjs‎

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
import fs from 'fs';
2+
import path from 'path';
3+
4+
function parseFrontMatter(fileContent) {
5+
const frontMatterRegex = /^---\n([\s\S]+?)\n---\n/;
6+
const match = fileContent.match(frontMatterRegex);
7+
8+
if (!match) {
9+
return { data: {}, content: fileContent };
10+
}
11+
12+
const frontMatterBlock = match[1];
13+
const content = fileContent.replace(frontMatterRegex, '');
14+
15+
const data = {};
16+
frontMatterBlock.split('\n').forEach((line) => {
17+
const parts = line.split(':');
18+
if (parts.length >= 2) {
19+
const key = parts[0].trim();
20+
let value = parts.slice(1).join(':').trim();
21+
if (
22+
(value.startsWith("'" ) && value.endsWith("'")) ||
23+
(value.startsWith('"') && value.endsWith('"'))
24+
) {
25+
value = value.slice(1, -1);
26+
}
27+
data[key] = value;
28+
}
29+
});
30+
31+
return { data, content };
32+
}
33+
34+
function processSidebar(
35+
items,
36+
docsPath,
37+
version,
38+
isLatest,
39+
baseUrl,
40+
level = 0
41+
) {
42+
let llmsContent = '';
43+
let fullDocsList = [];
44+
45+
items.forEach((item) => {
46+
if (typeof item === 'string') {
47+
const id = item;
48+
const filePath = path.join(docsPath, `${id}.md`);
49+
50+
if (fs.existsSync(filePath)) {
51+
const fileContent = fs.readFileSync(filePath, 'utf8');
52+
const { data, content } = parseFrontMatter(fileContent);
53+
54+
const title = data.title || id;
55+
const description = data.description || '';
56+
57+
let urlPath;
58+
if (isLatest) {
59+
urlPath = `/docs/${id}`;
60+
} else {
61+
urlPath = `/docs/${version}/${id}`;
62+
}
63+
64+
const fullUrl = `${baseUrl}${urlPath}`;
65+
66+
llmsContent += `- [${title}](${fullUrl})${
67+
description ? `: ${description}` : ''
68+
}\n`;
69+
70+
fullDocsList.push({
71+
title,
72+
url: fullUrl,
73+
content,
74+
});
75+
}
76+
} else if (item.type === 'category') {
77+
const label = item.label;
78+
const headingPrefix = '#'.repeat(level + 3);
79+
llmsContent += `\n${headingPrefix} ${label}\n\n`;
80+
81+
const { content: childContent, docs: childDocs } = processSidebar(
82+
item.items,
83+
docsPath,
84+
version,
85+
isLatest,
86+
baseUrl,
87+
level + 1
88+
);
89+
llmsContent += childContent;
90+
fullDocsList = fullDocsList.concat(childDocs);
91+
}
92+
});
93+
94+
return { content: llmsContent, docs: fullDocsList };
95+
}
96+
97+
function generateForVersion(
98+
siteDir,
99+
outDir,
100+
version,
101+
outputPrefix,
102+
isLatest,
103+
baseUrl
104+
) {
105+
console.log(`[generate-llms-txt] Generating for version ${version}...`);
106+
107+
const docsPath = path.join(siteDir, 'versioned_docs', `version-${version}`);
108+
const sidebarPath = path.join(
109+
siteDir,
110+
'versioned_sidebars',
111+
`version-${version}-sidebars.json`
112+
);
113+
114+
if (!fs.existsSync(sidebarPath)) {
115+
console.warn(
116+
`[generate-llms-txt] Sidebar not found: ${sidebarPath}, skipping.`
117+
);
118+
return;
119+
}
120+
121+
const sidebarConfig = JSON.parse(fs.readFileSync(sidebarPath, 'utf8'));
122+
let rootItems = sidebarConfig.docs || Object.values(sidebarConfig)[0] || [];
123+
124+
if (!Array.isArray(rootItems) && typeof rootItems === 'object') {
125+
const normalized = [];
126+
for (const [label, items] of Object.entries(rootItems)) {
127+
normalized.push({
128+
type: 'category',
129+
label: label,
130+
items: items,
131+
});
132+
}
133+
rootItems = normalized;
134+
}
135+
136+
const { content: sidebarContent, docs } = processSidebar(
137+
rootItems,
138+
docsPath,
139+
version,
140+
isLatest,
141+
baseUrl
142+
);
143+
144+
let llmsTxt = `# React Navigation ${version}\n\n`;
145+
llmsTxt += `> Routing and navigation for your React Native apps.\n\n`;
146+
llmsTxt += `## Documentation\n`;
147+
148+
llmsTxt += sidebarContent;
149+
150+
let llmsFullTxt = `# React Navigation ${version} Documentation\n\n`;
151+
152+
docs.forEach((doc) => {
153+
llmsFullTxt += `## ${doc.title}\n\n`;
154+
llmsFullTxt += `Source: ${doc.url}\n\n`;
155+
llmsFullTxt += `${doc.content.trim()}\n\n---\n\n`;
156+
});
157+
158+
const summaryFilename = `${outputPrefix}.txt`;
159+
160+
let fullFilename;
161+
if (outputPrefix === 'llms') {
162+
fullFilename = 'llms-full.txt';
163+
} else {
164+
if (outputPrefix.includes('llms-')) {
165+
fullFilename = outputPrefix.replace('llms-', 'llms-full-') + '.txt';
166+
} else {
167+
fullFilename = outputPrefix + '-full.txt';
168+
}
169+
}
170+
171+
fs.writeFileSync(path.join(outDir, summaryFilename), llmsTxt);
172+
console.log(`[generate-llms-txt] Wrote ${summaryFilename}`);
173+
174+
fs.writeFileSync(path.join(outDir, fullFilename), llmsFullTxt);
175+
console.log(`[generate-llms-txt] Wrote ${fullFilename}`);
176+
}
177+
178+
export default function (context, options) {
179+
return {
180+
name: 'generate-llms-txt',
181+
async postBuild({ siteDir, outDir }) {
182+
const { latestVersion, baseUrl } = options;
183+
184+
if (!latestVersion) {
185+
throw new Error(
186+
'[generate-llms-txt] "latestVersion" option is required.'
187+
);
188+
}
189+
if (!baseUrl) {
190+
throw new Error('[generate-llms-txt] "baseUrl" option is required.');
191+
}
192+
193+
const versionsPath = path.join(siteDir, 'versions.json');
194+
let versions = [];
195+
if (fs.existsSync(versionsPath)) {
196+
versions = JSON.parse(fs.readFileSync(versionsPath, 'utf8'));
197+
} else {
198+
console.warn(
199+
'[generate-llms-txt] versions.json not found. Generating only for latestVersion.'
200+
);
201+
versions = [latestVersion];
202+
}
203+
204+
console.log(`[generate-llms-txt] Found versions: ${versions.join(', ')}`);
205+
console.log(`[generate-llms-txt] Latest version: ${latestVersion}`);
206+
207+
versions.forEach((version) => {
208+
const isLatest = version === latestVersion;
209+
const outputPrefix = isLatest ? 'llms' : `llms-v${version}`;
210+
211+
generateForVersion(
212+
siteDir,
213+
outDir,
214+
version,
215+
outputPrefix,
216+
isLatest,
217+
baseUrl
218+
);
219+
});
220+
},
221+
};
222+
}

0 commit comments

Comments
 (0)