Add data pipeline: fetch and parse Cheat mod DBs into JSON
Pulls items/perks/buffs/skills CSVs and command docs from pryans/kcd2-cheat, parses the mod's CSV dialect (backslash-escaped commas, multiline quoted fields) and BBCode command docs into vendored JSON under src/data/. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Homer
@@ -0,0 +1,181 @@
|
||||
// Regenerates src/data/*.json from the KCD2 "Cheat" mod repository
|
||||
// (https://github.com/pryans/kcd2-cheat — source of the Nexus "Cheat" mod).
|
||||
//
|
||||
// Usage: node scripts/fetch-data.mjs
|
||||
//
|
||||
// Output is committed to the repo so builds are offline; re-run to refresh
|
||||
// when the mod updates its item/perk/buff/skill dumps or command docs.
|
||||
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const BASE = 'https://raw.githubusercontent.com/pryans/kcd2-cheat/main/docs/';
|
||||
const OUT_DIR = join(dirname(fileURLToPath(import.meta.url)), '..', 'src', 'data');
|
||||
|
||||
async function fetchText(name) {
|
||||
const res = await fetch(BASE + name);
|
||||
if (!res.ok) throw new Error(`Fetching ${name} failed: HTTP ${res.status}`);
|
||||
return (await res.text()).replaceAll('\r\n', '\n');
|
||||
}
|
||||
|
||||
// The mod's CSV dialect: fields are double-quoted (header and empty trailing
|
||||
// fields are bare), commas inside fields are backslash-escaped, and quoted
|
||||
// fields may contain literal newlines.
|
||||
function parseCsv(text) {
|
||||
const rows = [];
|
||||
let row = [];
|
||||
let field = '';
|
||||
let inQuotes = false;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const c = text[i];
|
||||
if (inQuotes) {
|
||||
if (c === '\\' && i + 1 < text.length) {
|
||||
field += text[++i];
|
||||
} else if (c === '"') {
|
||||
inQuotes = false;
|
||||
} else {
|
||||
field += c;
|
||||
}
|
||||
} else if (c === '"') {
|
||||
inQuotes = true;
|
||||
} else if (c === ',') {
|
||||
row.push(field);
|
||||
field = '';
|
||||
} else if (c === '\n') {
|
||||
row.push(field);
|
||||
if (row.some((f) => f !== '')) rows.push(row);
|
||||
row = [];
|
||||
field = '';
|
||||
} else {
|
||||
field += c;
|
||||
}
|
||||
}
|
||||
row.push(field);
|
||||
if (row.some((f) => f !== '')) rows.push(row);
|
||||
return rows.slice(1); // drop header
|
||||
}
|
||||
|
||||
// Skill/perk descriptions are HTML-entity-encoded rich text; flatten to plain
|
||||
// text with paragraph breaks.
|
||||
function cleanText(s) {
|
||||
return s
|
||||
.replaceAll('<', '<')
|
||||
.replaceAll('>', '>')
|
||||
.replaceAll('"', '"')
|
||||
.replaceAll(''', "'")
|
||||
.replaceAll(''', "'")
|
||||
.replaceAll('&', '&')
|
||||
.replace(/<\/p>\s*<p>/g, '\n\n')
|
||||
.replace(/<br\s*\/?>/g, '\n')
|
||||
.replace(/<[^>]+>/g, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
const CATEGORY_RULES = [
|
||||
[/money/, 'Money'],
|
||||
[/item|inventory|stash|gear|stolen/, 'Items & Inventory'],
|
||||
[/skill|stat/, 'Skills & Stats'],
|
||||
[/perk/, 'Perks'],
|
||||
[/buff|potion/, 'Buffs'],
|
||||
[/horse/, 'Horse'],
|
||||
[/teleport|_loc$|clip|phys_|checkpoint/, 'Movement & Teleport'],
|
||||
[/npc|kill|spawn|charm|target|revive/, 'NPCs'],
|
||||
[/time|weather|map/, 'World & Time'],
|
||||
[/lockpick|pickpocket/, 'Minigames'],
|
||||
[/hud|compass|statusbar|reticle|third_person|regen|wash|save|state/, 'Player & UI'],
|
||||
[/action|alias|localization|find_/, 'Utility'],
|
||||
];
|
||||
|
||||
function categorize(name) {
|
||||
for (const [re, cat] of CATEGORY_RULES) if (re.test(name)) return cat;
|
||||
return 'Misc';
|
||||
}
|
||||
|
||||
function stripBbcode(s) {
|
||||
return s.replace(/\[\/?(?:b|i|u|size|color|url)(?:=[^\]]*)?\]/g, '').trim();
|
||||
}
|
||||
|
||||
// docs.txt: intro + command index, then one block per command:
|
||||
// [size=4][b]name[/b][/size]
|
||||
// description lines
|
||||
// [b]Arguments:[/b] (optional section, tab-indented "arg: (required type) desc")
|
||||
// [b]Examples:[/b] (optional section, tab-indented caption/command groups)
|
||||
function parseCommandDocs(text) {
|
||||
const version = text.match(/Cheat version ([\d.]+)/)?.[1] ?? null;
|
||||
const commands = [];
|
||||
const blockRe = /^\[size=4\]\[b\](\w+)\[\/b\]\[\/size\]$/gm;
|
||||
const headers = [...text.matchAll(blockRe)];
|
||||
for (let h = 0; h < headers.length; h++) {
|
||||
const name = headers[h][1];
|
||||
const start = headers[h].index + headers[h][0].length;
|
||||
const end = h + 1 < headers.length ? headers[h + 1].index : text.length;
|
||||
const lines = text.slice(start, end).split('\n');
|
||||
|
||||
let section = 'desc';
|
||||
const descLines = [];
|
||||
const args = [];
|
||||
const examples = [];
|
||||
let example = null;
|
||||
for (const rawLine of lines) {
|
||||
const line = rawLine.replace(/\s+$/, '');
|
||||
if (/^\[b\]Arguments:\[\/b\]$/.test(line)) { section = 'args'; continue; }
|
||||
if (/^\[b\]Examples:\[\/b\]$/.test(line)) { section = 'examples'; continue; }
|
||||
if (section === 'desc') {
|
||||
descLines.push(stripBbcode(line));
|
||||
} else if (section === 'args') {
|
||||
const m = line.match(/^\t(\w+): \((required|optional)(?: (\w+))?\)\s*(.*)$/);
|
||||
if (m) args.push({ name: m[1], required: m[2] === 'required', type: m[3] ?? 'string', desc: m[4] });
|
||||
} else if (section === 'examples') {
|
||||
const t = line.replace(/^\t/, '');
|
||||
if (t === '') {
|
||||
if (example?.command) { examples.push(example); example = null; }
|
||||
} else if (t.startsWith(name)) {
|
||||
example ??= { caption: '' };
|
||||
example.command = t;
|
||||
} else {
|
||||
example ??= { caption: '' };
|
||||
example.caption = (example.caption ? example.caption + ' ' : '') + t.replace(/:$/, '');
|
||||
}
|
||||
}
|
||||
}
|
||||
if (example?.command) examples.push(example);
|
||||
commands.push({
|
||||
name,
|
||||
desc: descLines.join('\n').replace(/\n{3,}/g, '\n\n').trim(),
|
||||
args,
|
||||
examples,
|
||||
category: categorize(name),
|
||||
source: 'cheatmod',
|
||||
});
|
||||
}
|
||||
return { version, commands };
|
||||
}
|
||||
|
||||
async function writeJson(name, data) {
|
||||
await writeFile(join(OUT_DIR, name), JSON.stringify(data));
|
||||
console.log(`${name}: ${Array.isArray(data) ? data.length : data.commands?.length ?? ''} records`);
|
||||
}
|
||||
|
||||
await mkdir(OUT_DIR, { recursive: true });
|
||||
|
||||
const [itemsCsv, perksCsv, buffsCsv, skillsCsv, docsTxt] = await Promise.all(
|
||||
['items.csv', 'perks.csv', 'buffs.csv', 'skills.csv', 'docs.txt'].map(fetchText),
|
||||
);
|
||||
|
||||
const items = parseCsv(itemsCsv).map(([id, name, desc]) => ({ id, name, desc: cleanText(desc ?? '') }));
|
||||
const perks = parseCsv(perksCsv).map(([id, name, desc]) => ({ id, name, desc: cleanText(desc ?? '') }));
|
||||
const buffs = parseCsv(buffsCsv).map(([id, name, params, desc]) => ({ id, name, params: params ?? '', desc: cleanText(desc ?? '') }));
|
||||
const skills = parseCsv(skillsCsv).map(([id, name, desc]) => ({ id, name, desc: cleanText(desc ?? '') }));
|
||||
const { version, commands } = parseCommandDocs(docsTxt);
|
||||
|
||||
await writeJson('items.json', items);
|
||||
await writeJson('perks.json', perks);
|
||||
await writeJson('buffs.json', buffs);
|
||||
await writeJson('skills.json', skills);
|
||||
await writeJson('commands-mod.json', commands);
|
||||
await writeJson('meta.json', {
|
||||
source: 'https://github.com/pryans/kcd2-cheat',
|
||||
cheatModDocsVersion: version,
|
||||
fetchedAt: new Date().toISOString(),
|
||||
});
|
||||
Reference in New Issue
Block a user