a07522a189
React 19 SPA + node:http/node:sqlite API in one container. Card data is baked into the image from a Scryfall bulk build, so the runtime never calls Scryfall. No server-side user state: decks live in the browser and travel in the share URL, which is why there is no PVC and replicas: 1 is correct. Adds a README covering the architecture and the reasoning behind it, and publishes the project skills under .claude/skills/. Co-Authored-By: Claude Code <noreply@anthropic.com>
382 lines
14 KiB
JavaScript
382 lines
14 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Build the Commander-legal card index.
|
|
*
|
|
* Downloads the Scryfall "Default Cards" bulk JSONL (one object per distinct
|
|
* printed card), keeps every card whose `legalities.commander` is `legal`,
|
|
* and writes `data/cards.db` — a SQLite file the API serves read-only.
|
|
*
|
|
* Why bulk at build time rather than the live API:
|
|
* - the API forbids browser CORS and rate-limits to ~10 req/s, which a
|
|
* faceted card browser cannot live within;
|
|
* - Commander legality, colour identity and ban status are all static
|
|
* between releases, so there is nothing to re-fetch at runtime.
|
|
*
|
|
* The Scryfall bulk catalogue is the source of truth for the download URL;
|
|
* filenames are versioned and change on every regeneration, so they are
|
|
* never hardcoded here.
|
|
*
|
|
* node scripts/build-cards.mjs [--out data/cards.db] [--all]
|
|
*
|
|
* --all keep non-legal cards too (diagnostics and format-browsing)
|
|
*/
|
|
|
|
import { createReadStream, createWriteStream, existsSync } from 'node:fs'
|
|
import { mkdir, rm, rename, stat } from 'node:fs/promises'
|
|
import { createGunzip } from 'node:zlib'
|
|
import { Readable } from 'node:stream'
|
|
import { pipeline } from 'node:stream/promises'
|
|
import { DatabaseSync } from 'node:sqlite'
|
|
import { dirname, resolve } from 'node:path'
|
|
import { fileURLToPath } from 'node:url'
|
|
|
|
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..')
|
|
const UA = 'magik-commander-builder/1.0 (https://magic.marcospaulo.dev.br)'
|
|
const BULK_CATALOG = 'https://api.scryfall.com/bulk-data'
|
|
const BULK_TYPE = 'default_cards'
|
|
|
|
const args = process.argv.slice(2)
|
|
const outPath = resolve(ROOT, valueOf('--out') ?? 'data/cards.db')
|
|
const keepAll = args.includes('--all')
|
|
|
|
function valueOf(flag) {
|
|
const i = args.indexOf(flag)
|
|
return i === -1 ? undefined : args[i + 1]
|
|
}
|
|
|
|
const SCHEMA = `
|
|
PRAGMA journal_mode = OFF;
|
|
PRAGMA synchronous = OFF;
|
|
|
|
CREATE TABLE cards (
|
|
id TEXT PRIMARY KEY, -- Scryfall id of the chosen printing
|
|
oracle_id TEXT NOT NULL, -- stable across printings
|
|
name TEXT NOT NULL,
|
|
mana_cost TEXT NOT NULL DEFAULT '',
|
|
cmc INTEGER NOT NULL DEFAULT 0,
|
|
mv_cap INTEGER NOT NULL DEFAULT 0, -- cmc bucket for the 7+ histogram
|
|
type_line TEXT NOT NULL DEFAULT '',
|
|
oracle_text TEXT NOT NULL DEFAULT '',
|
|
color_identity TEXT NOT NULL DEFAULT '',
|
|
colors TEXT NOT NULL DEFAULT '',
|
|
rarity TEXT NOT NULL DEFAULT '',
|
|
power TEXT,
|
|
toughness TEXT,
|
|
loyalty TEXT,
|
|
layout TEXT NOT NULL DEFAULT '',
|
|
faces TEXT, -- JSON array for multi-face cards
|
|
released_at TEXT NOT NULL DEFAULT '',
|
|
artist TEXT NOT NULL DEFAULT '',
|
|
set_name TEXT NOT NULL DEFAULT '',
|
|
set_code TEXT NOT NULL DEFAULT '',
|
|
commander_legality TEXT NOT NULL, -- legal | banned | not_legal
|
|
commander_eligible INTEGER NOT NULL DEFAULT 0,
|
|
is_partner INTEGER NOT NULL DEFAULT 0,
|
|
is_basic_land INTEGER NOT NULL DEFAULT 0,
|
|
legal_formats TEXT NOT NULL DEFAULT '', -- JSON map, formats where legal
|
|
image_small TEXT, image_normal TEXT, image_large TEXT,
|
|
image_png TEXT, image_art_crop TEXT, image_board_crop TEXT
|
|
);
|
|
|
|
CREATE INDEX idx_cards_name ON cards(name);
|
|
CREATE INDEX idx_cards_commander ON cards(commander_eligible);
|
|
CREATE INDEX idx_cards_cmc ON cards(cmc);
|
|
CREATE INDEX idx_cards_rarity ON cards(rarity);
|
|
CREATE INDEX idx_cards_identity ON cards(color_identity);
|
|
`
|
|
|
|
/**
|
|
* SQLite has no `LIKE` index, so a search over ~30k rows is a table scan of
|
|
* a single narrow table — a few milliseconds. FTS5 is not used: its
|
|
* tokenisation hurts name-prefix matching ("sol" must find "Sol Ring"), which
|
|
* is the dominant query in a card browser.
|
|
*/
|
|
async function main() {
|
|
const { url, size: expectedSize } = await resolveBulkUrl()
|
|
console.log(`bulk: ${url}`)
|
|
|
|
await mkdir(dirname(outPath), { recursive: true })
|
|
const tmpPath = `${outPath}.tmp`
|
|
await rm(tmpPath, { force: true })
|
|
|
|
const db = new DatabaseSync(tmpPath)
|
|
db.exec(SCHEMA)
|
|
|
|
const insert = db.prepare(`
|
|
INSERT INTO cards (
|
|
id, oracle_id, name, mana_cost, cmc, mv_cap, type_line, oracle_text,
|
|
color_identity, colors, rarity, power, toughness, loyalty, layout, faces,
|
|
released_at, artist, set_name, set_code, commander_legality,
|
|
commander_eligible, is_partner, is_basic_land, legal_formats,
|
|
image_small, image_normal, image_large, image_png, image_art_crop, image_board_crop
|
|
) VALUES (
|
|
?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?
|
|
)
|
|
`)
|
|
|
|
db.exec('BEGIN')
|
|
|
|
// The bulk file holds one object per card *printing*, and digital-only
|
|
// reprints (Alchemy, Vintage Masters) are marked banned in paper Commander.
|
|
// Indexing those verbatim would show Ancestral Recall eleven times, all
|
|
// "banned", which is a printing-level fact masquerading as a card-level
|
|
// one. So printings are collapsed to one row per oracle card, keeping the
|
|
// best paper-legal one.
|
|
const best = new Map()
|
|
let seen = 0
|
|
|
|
for await (const card of readBulk(url, expectedSize)) {
|
|
seen++
|
|
const legality = card.legalities?.commander
|
|
if (legality === undefined) continue
|
|
// Reversible and split cards publish bare faces as their own objects with
|
|
// no type line. The front face is indexed under the full card, so these
|
|
// would only ever add empty duplicates.
|
|
if ((card.type_line ?? '') === '') continue
|
|
// Banned cards are kept deliberately: the validator has to be able to
|
|
// name the offending card, and the commander picker has to be able to
|
|
// explain why a legendary card is unavailable.
|
|
if (!keepAll && legality !== 'legal' && legality !== 'banned') continue
|
|
|
|
const oracleId = card.oracle_id ?? card.id
|
|
const candidate = { row: project(card, legality), rank: rankPrinting(card, legality) }
|
|
const current = best.get(oracleId)
|
|
if (!current || compareRank(candidate.rank, current.rank) > 0) best.set(oracleId, candidate)
|
|
|
|
if (seen % 20000 === 0) console.log(` ${seen} read · ${best.size} distinct cards`)
|
|
}
|
|
|
|
let kept = 0
|
|
for (const { row } of best.values()) {
|
|
insert.run(...row)
|
|
kept++
|
|
}
|
|
|
|
db.exec('COMMIT')
|
|
db.exec('VACUUM')
|
|
db.exec('ANALYZE')
|
|
db.close()
|
|
|
|
await rename(tmpPath, outPath)
|
|
const { size } = await stat(outPath)
|
|
console.log(`done: ${kept} distinct cards from ${seen} printings -> ${outPath} (${(size / 1e6).toFixed(1)} MB)`)
|
|
}
|
|
|
|
const SET_TYPE_RANK = {
|
|
core: 9,
|
|
commander: 8,
|
|
draft_innovation: 7,
|
|
promo: 6,
|
|
masters: 5,
|
|
expansion: 4,
|
|
starter: 3,
|
|
limited: 2,
|
|
token: 0,
|
|
}
|
|
|
|
/**
|
|
* Orders printings of the same card so the most useful one wins.
|
|
*
|
|
* Paper legality comes first: Commander is played with paper cards, so a
|
|
* printing Scryfall marks banned there is not a ban on the card. Within the
|
|
* legal group, prefer a core-set printing in English, then the most recent
|
|
* release for its best available art.
|
|
*/
|
|
function rankPrinting(card, legality) {
|
|
return [
|
|
legality === 'legal' ? 1 : 0,
|
|
card.set_code ? 1 : 0,
|
|
card.lang === 'en' ? 1 : 0,
|
|
SET_TYPE_RANK[card.set_type] ?? 1,
|
|
card.released_at ?? '',
|
|
]
|
|
}
|
|
|
|
function compareRank(a, b) {
|
|
for (let i = 0; i < a.length; i++) {
|
|
if (a[i] !== b[i]) return a[i] > b[i] ? 1 : -1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
async function resolveBulkUrl() {
|
|
const res = await fetch(BULK_CATALOG, { headers: { 'User-Agent': UA, Accept: 'application/json' } })
|
|
if (!res.ok) throw new Error(`bulk catalogue ${res.status}`)
|
|
const { data } = await res.json()
|
|
const entry = data.find((e) => e.type === BULK_TYPE)
|
|
if (!entry?.jsonl_download_uri) {
|
|
throw new Error(`no "${BULK_TYPE}" entry with a download uri in the bulk catalogue`)
|
|
}
|
|
return { url: entry.jsonl_download_uri, size: entry.compressed_size }
|
|
}
|
|
|
|
/**
|
|
* Yields one parsed card object at a time from the bulk download.
|
|
*
|
|
* The archive is cached under `data/.cache/` because it is versioned per
|
|
* Scryfall release: rebuilding the index after a schema tweak should not
|
|
* re-fetch 200 MB.
|
|
*
|
|
* The cache is size-checked against the bulk catalogue on every use. A
|
|
* download interrupted part-way through still un-gzips, still yields valid
|
|
* JSON lines for everything before the break, and still produces a plausible
|
|
* looking index — one that is silently missing every card Scryfall listed
|
|
* after the cut. That happened once here and cost an afternoon: the only
|
|
* defence is to compare the byte count against the size Scryfall published.
|
|
*
|
|
* Lines are split by hand rather than with `readline` so that a transport or
|
|
* gunzip error propagates out of the `for await` instead of being swallowed
|
|
* by the reader's event emitter.
|
|
*/
|
|
async function* readBulk(url, expectedSize) {
|
|
const cachePath = resolve(ROOT, 'data/.cache', url.split('/').pop())
|
|
await mkdir(dirname(cachePath), { recursive: true })
|
|
|
|
const incomplete = (actual) =>
|
|
`${cachePath} is ${actual} bytes, Scryfall says the file is ${expectedSize}; it is a partial download`
|
|
|
|
let size = existsSync(cachePath) ? (await stat(cachePath)).size : 0
|
|
if (size !== 0 && size !== expectedSize) {
|
|
await rm(cachePath, { force: true })
|
|
size = 0
|
|
}
|
|
|
|
if (size === 0) {
|
|
console.log('downloading…')
|
|
const res = await fetch(url, { headers: { 'User-Agent': UA } })
|
|
if (!res.ok || !res.body) throw new Error(`bulk download ${res.status} ${url}`)
|
|
const partial = `${cachePath}.part`
|
|
await pipeline(Readable.fromWeb(res.body), createWriteStream(partial))
|
|
const { size: written } = await stat(partial)
|
|
if (written !== expectedSize) {
|
|
await rm(partial, { force: true })
|
|
throw new Error(incomplete(written))
|
|
}
|
|
await rename(partial, cachePath)
|
|
} else {
|
|
console.log(`using cached ${cachePath}`)
|
|
}
|
|
|
|
const gunzip = createReadStream(cachePath).pipe(createGunzip())
|
|
let rest = ''
|
|
for await (const chunk of gunzip) {
|
|
rest += chunk
|
|
let nl
|
|
while ((nl = rest.indexOf('\n')) !== -1) {
|
|
if (rest.slice(0, nl) !== '') yield JSON.parse(rest.slice(0, nl))
|
|
rest = rest.slice(nl + 1)
|
|
}
|
|
}
|
|
if (rest.trim() !== '') yield JSON.parse(rest)
|
|
}
|
|
|
|
/**
|
|
* Commander eligibility, per CR 903.3:
|
|
* (a) a legendary creature
|
|
* (b) a legendary Vehicle
|
|
* (c) a legendary Spacecraft with printed power/toughness
|
|
* 903.3a — any legendary card whose text says it can be your commander
|
|
*
|
|
* 903.3a is the awkward one: Scryfall and MTGJSON both strip that sentence
|
|
* from the card text they publish, so it cannot be read card by card from any
|
|
* accessible dataset. Wizards' own search indexes only the "can be your
|
|
* commander" wording and misses the rest, and its Partner filter omits every
|
|
* Partner planeswalker.
|
|
*
|
|
* The two classes that the stripped sentence would otherwise decide are
|
|
* therefore taken from the type line, which is correct for all of them:
|
|
*
|
|
* - Legendary Planeswalkers. Every one printed since the 2017 change that
|
|
* made planeswalkers legal commanders carries the ability, and the
|
|
* pre-2017 ones were reprinted with it.
|
|
* - Legendary Vehicles and Spacecraft, which 903.3b/c admit by type.
|
|
*
|
|
* What neither type line nor any dataset can supply is the handful of
|
|
* legendary artifacts, enchantments, lands and sorceries that lead a deck
|
|
* only because of 903.3a. That is a short, closed list, kept in
|
|
* DESIGNATED_COMMANDERS below. Add to it when a new set introduces one.
|
|
*/
|
|
const DESIGNATED_COMMANDERS = new Set([
|
|
'Agatha\'s Soul Cauldron',
|
|
'Gerrard\'s Hourglass Pendant',
|
|
'Instrument of the Bards',
|
|
'Lithoform Engine',
|
|
'Nyx Lotus',
|
|
'Ozolith, the Shattered Spire',
|
|
'The Book of Exalted Deeds',
|
|
'The Cauldron of Eternity',
|
|
'The Earth Crystal',
|
|
'The Mightstone and Weakstone',
|
|
"Urza's Sylex",
|
|
])
|
|
|
|
/** A Background is a legendary creature but can never lead a deck alone. */
|
|
function isCommanderEligible(card) {
|
|
const typeLine = card.type_line ?? ''
|
|
if (!/legendary/i.test(typeLine)) return false
|
|
if (/\bbackground\b/i.test(typeLine)) return false
|
|
if (/creature/i.test(typeLine) || /vehicle/i.test(typeLine) || /\bplaneswalker\b/i.test(typeLine)) return true
|
|
if (/\bspacecraft\b/i.test(typeLine)) return card.power != null && card.toughness != null
|
|
return (
|
|
DESIGNATED_COMMANDERS.has((card.name ?? '').split(' // ')[0]) ||
|
|
/can be your commander/i.test(card.oracle_text ?? '')
|
|
)
|
|
}
|
|
|
|
function project(card, commanderLegality) {
|
|
const typeLine = card.type_line ?? ''
|
|
const oracleText = card.oracle_text ?? ''
|
|
const name = (card.name ?? '').split(' // ')[0]
|
|
const formats = []
|
|
for (const [format, status] of Object.entries(card.legalities ?? {})) {
|
|
if (status === 'legal') formats.push(format)
|
|
}
|
|
|
|
const cmc = Number.isFinite(card.cmc) ? card.cmc : 0
|
|
// Scryfall models both of these as arrays; a colourless card has an empty
|
|
// one, which must not be bound to a TEXT column.
|
|
const identity = (card.color_identity ?? []).join('')
|
|
const colors = (card.colors ?? []).join('')
|
|
|
|
const commanderEligible = isCommanderEligible(card)
|
|
|
|
return [
|
|
card.id,
|
|
card.oracle_id ?? card.id,
|
|
name,
|
|
card.mana_cost ?? '',
|
|
cmc,
|
|
Math.min(cmc, 7),
|
|
typeLine,
|
|
oracleText,
|
|
identity,
|
|
colors,
|
|
card.rarity ?? '',
|
|
card.power ?? null,
|
|
card.toughness ?? null,
|
|
card.loyalty ?? null,
|
|
card.layout ?? '',
|
|
card.faces ? JSON.stringify(card.faces) : null,
|
|
card.released_at ?? '',
|
|
card.artist ?? '',
|
|
card.set_name ?? '',
|
|
card.set_code ?? '',
|
|
commanderLegality,
|
|
commanderEligible ? 1 : 0,
|
|
/\bpartner\b/i.test(oracleText) ? 1 : 0,
|
|
/^basic (snow )?land\b/i.test(typeLine.trim()) ? 1 : 0,
|
|
JSON.stringify(formats),
|
|
card.image_uris?.small ?? null,
|
|
card.image_uris?.normal ?? null,
|
|
card.image_uris?.large ?? null,
|
|
card.image_uris?.png ?? null,
|
|
card.image_uris?.art_crop ?? null,
|
|
card.image_uris?.board_crop ?? null,
|
|
]
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error(err)
|
|
process.exitCode = 1
|
|
})
|