Files
magik/scripts/build-cards.mjs
masi a07522a189 MAGIK — Commander deck builder
React 19 SPA + node:http/node:sqlite API in one container. Card data is
baked into the image from a Scryfall bulk build, so the runtime never calls
Scryfall. No server-side user state: decks live in the browser and travel in
the share URL, which is why there is no PVC and replicas: 1 is correct.

Adds a README covering the architecture and the reasoning behind it, and
publishes the project skills under .claude/skills/.

Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-10-01 21:12:19 -03:00

382 lines
14 KiB
JavaScript

#!/usr/bin/env node
/**
* Build the Commander-legal card index.
*
* Downloads the Scryfall "Default Cards" bulk JSONL (one object per distinct
* printed card), keeps every card whose `legalities.commander` is `legal`,
* and writes `data/cards.db` — a SQLite file the API serves read-only.
*
* Why bulk at build time rather than the live API:
* - the API forbids browser CORS and rate-limits to ~10 req/s, which a
* faceted card browser cannot live within;
* - Commander legality, colour identity and ban status are all static
* between releases, so there is nothing to re-fetch at runtime.
*
* The Scryfall bulk catalogue is the source of truth for the download URL;
* filenames are versioned and change on every regeneration, so they are
* never hardcoded here.
*
* node scripts/build-cards.mjs [--out data/cards.db] [--all]
*
* --all keep non-legal cards too (diagnostics and format-browsing)
*/
import { createReadStream, createWriteStream, existsSync } from 'node:fs'
import { mkdir, rm, rename, stat } from 'node:fs/promises'
import { createGunzip } from 'node:zlib'
import { Readable } from 'node:stream'
import { pipeline } from 'node:stream/promises'
import { DatabaseSync } from 'node:sqlite'
import { dirname, resolve } from 'node:path'
import { fileURLToPath } from 'node:url'
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..')
const UA = 'magik-commander-builder/1.0 (https://magic.marcospaulo.dev.br)'
const BULK_CATALOG = 'https://api.scryfall.com/bulk-data'
const BULK_TYPE = 'default_cards'
const args = process.argv.slice(2)
const outPath = resolve(ROOT, valueOf('--out') ?? 'data/cards.db')
const keepAll = args.includes('--all')
function valueOf(flag) {
const i = args.indexOf(flag)
return i === -1 ? undefined : args[i + 1]
}
const SCHEMA = `
PRAGMA journal_mode = OFF;
PRAGMA synchronous = OFF;
CREATE TABLE cards (
id TEXT PRIMARY KEY, -- Scryfall id of the chosen printing
oracle_id TEXT NOT NULL, -- stable across printings
name TEXT NOT NULL,
mana_cost TEXT NOT NULL DEFAULT '',
cmc INTEGER NOT NULL DEFAULT 0,
mv_cap INTEGER NOT NULL DEFAULT 0, -- cmc bucket for the 7+ histogram
type_line TEXT NOT NULL DEFAULT '',
oracle_text TEXT NOT NULL DEFAULT '',
color_identity TEXT NOT NULL DEFAULT '',
colors TEXT NOT NULL DEFAULT '',
rarity TEXT NOT NULL DEFAULT '',
power TEXT,
toughness TEXT,
loyalty TEXT,
layout TEXT NOT NULL DEFAULT '',
faces TEXT, -- JSON array for multi-face cards
released_at TEXT NOT NULL DEFAULT '',
artist TEXT NOT NULL DEFAULT '',
set_name TEXT NOT NULL DEFAULT '',
set_code TEXT NOT NULL DEFAULT '',
commander_legality TEXT NOT NULL, -- legal | banned | not_legal
commander_eligible INTEGER NOT NULL DEFAULT 0,
is_partner INTEGER NOT NULL DEFAULT 0,
is_basic_land INTEGER NOT NULL DEFAULT 0,
legal_formats TEXT NOT NULL DEFAULT '', -- JSON map, formats where legal
image_small TEXT, image_normal TEXT, image_large TEXT,
image_png TEXT, image_art_crop TEXT, image_board_crop TEXT
);
CREATE INDEX idx_cards_name ON cards(name);
CREATE INDEX idx_cards_commander ON cards(commander_eligible);
CREATE INDEX idx_cards_cmc ON cards(cmc);
CREATE INDEX idx_cards_rarity ON cards(rarity);
CREATE INDEX idx_cards_identity ON cards(color_identity);
`
/**
* SQLite has no `LIKE` index, so a search over ~30k rows is a table scan of
* a single narrow table — a few milliseconds. FTS5 is not used: its
* tokenisation hurts name-prefix matching ("sol" must find "Sol Ring"), which
* is the dominant query in a card browser.
*/
async function main() {
const { url, size: expectedSize } = await resolveBulkUrl()
console.log(`bulk: ${url}`)
await mkdir(dirname(outPath), { recursive: true })
const tmpPath = `${outPath}.tmp`
await rm(tmpPath, { force: true })
const db = new DatabaseSync(tmpPath)
db.exec(SCHEMA)
const insert = db.prepare(`
INSERT INTO cards (
id, oracle_id, name, mana_cost, cmc, mv_cap, type_line, oracle_text,
color_identity, colors, rarity, power, toughness, loyalty, layout, faces,
released_at, artist, set_name, set_code, commander_legality,
commander_eligible, is_partner, is_basic_land, legal_formats,
image_small, image_normal, image_large, image_png, image_art_crop, image_board_crop
) VALUES (
?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?
)
`)
db.exec('BEGIN')
// The bulk file holds one object per card *printing*, and digital-only
// reprints (Alchemy, Vintage Masters) are marked banned in paper Commander.
// Indexing those verbatim would show Ancestral Recall eleven times, all
// "banned", which is a printing-level fact masquerading as a card-level
// one. So printings are collapsed to one row per oracle card, keeping the
// best paper-legal one.
const best = new Map()
let seen = 0
for await (const card of readBulk(url, expectedSize)) {
seen++
const legality = card.legalities?.commander
if (legality === undefined) continue
// Reversible and split cards publish bare faces as their own objects with
// no type line. The front face is indexed under the full card, so these
// would only ever add empty duplicates.
if ((card.type_line ?? '') === '') continue
// Banned cards are kept deliberately: the validator has to be able to
// name the offending card, and the commander picker has to be able to
// explain why a legendary card is unavailable.
if (!keepAll && legality !== 'legal' && legality !== 'banned') continue
const oracleId = card.oracle_id ?? card.id
const candidate = { row: project(card, legality), rank: rankPrinting(card, legality) }
const current = best.get(oracleId)
if (!current || compareRank(candidate.rank, current.rank) > 0) best.set(oracleId, candidate)
if (seen % 20000 === 0) console.log(` ${seen} read · ${best.size} distinct cards`)
}
let kept = 0
for (const { row } of best.values()) {
insert.run(...row)
kept++
}
db.exec('COMMIT')
db.exec('VACUUM')
db.exec('ANALYZE')
db.close()
await rename(tmpPath, outPath)
const { size } = await stat(outPath)
console.log(`done: ${kept} distinct cards from ${seen} printings -> ${outPath} (${(size / 1e6).toFixed(1)} MB)`)
}
const SET_TYPE_RANK = {
core: 9,
commander: 8,
draft_innovation: 7,
promo: 6,
masters: 5,
expansion: 4,
starter: 3,
limited: 2,
token: 0,
}
/**
* Orders printings of the same card so the most useful one wins.
*
* Paper legality comes first: Commander is played with paper cards, so a
* printing Scryfall marks banned there is not a ban on the card. Within the
* legal group, prefer a core-set printing in English, then the most recent
* release for its best available art.
*/
function rankPrinting(card, legality) {
return [
legality === 'legal' ? 1 : 0,
card.set_code ? 1 : 0,
card.lang === 'en' ? 1 : 0,
SET_TYPE_RANK[card.set_type] ?? 1,
card.released_at ?? '',
]
}
function compareRank(a, b) {
for (let i = 0; i < a.length; i++) {
if (a[i] !== b[i]) return a[i] > b[i] ? 1 : -1
}
return 0
}
async function resolveBulkUrl() {
const res = await fetch(BULK_CATALOG, { headers: { 'User-Agent': UA, Accept: 'application/json' } })
if (!res.ok) throw new Error(`bulk catalogue ${res.status}`)
const { data } = await res.json()
const entry = data.find((e) => e.type === BULK_TYPE)
if (!entry?.jsonl_download_uri) {
throw new Error(`no "${BULK_TYPE}" entry with a download uri in the bulk catalogue`)
}
return { url: entry.jsonl_download_uri, size: entry.compressed_size }
}
/**
* Yields one parsed card object at a time from the bulk download.
*
* The archive is cached under `data/.cache/` because it is versioned per
* Scryfall release: rebuilding the index after a schema tweak should not
* re-fetch 200 MB.
*
* The cache is size-checked against the bulk catalogue on every use. A
* download interrupted part-way through still un-gzips, still yields valid
* JSON lines for everything before the break, and still produces a plausible
* looking index — one that is silently missing every card Scryfall listed
* after the cut. That happened once here and cost an afternoon: the only
* defence is to compare the byte count against the size Scryfall published.
*
* Lines are split by hand rather than with `readline` so that a transport or
* gunzip error propagates out of the `for await` instead of being swallowed
* by the reader's event emitter.
*/
async function* readBulk(url, expectedSize) {
const cachePath = resolve(ROOT, 'data/.cache', url.split('/').pop())
await mkdir(dirname(cachePath), { recursive: true })
const incomplete = (actual) =>
`${cachePath} is ${actual} bytes, Scryfall says the file is ${expectedSize}; it is a partial download`
let size = existsSync(cachePath) ? (await stat(cachePath)).size : 0
if (size !== 0 && size !== expectedSize) {
await rm(cachePath, { force: true })
size = 0
}
if (size === 0) {
console.log('downloading…')
const res = await fetch(url, { headers: { 'User-Agent': UA } })
if (!res.ok || !res.body) throw new Error(`bulk download ${res.status} ${url}`)
const partial = `${cachePath}.part`
await pipeline(Readable.fromWeb(res.body), createWriteStream(partial))
const { size: written } = await stat(partial)
if (written !== expectedSize) {
await rm(partial, { force: true })
throw new Error(incomplete(written))
}
await rename(partial, cachePath)
} else {
console.log(`using cached ${cachePath}`)
}
const gunzip = createReadStream(cachePath).pipe(createGunzip())
let rest = ''
for await (const chunk of gunzip) {
rest += chunk
let nl
while ((nl = rest.indexOf('\n')) !== -1) {
if (rest.slice(0, nl) !== '') yield JSON.parse(rest.slice(0, nl))
rest = rest.slice(nl + 1)
}
}
if (rest.trim() !== '') yield JSON.parse(rest)
}
/**
* Commander eligibility, per CR 903.3:
* (a) a legendary creature
* (b) a legendary Vehicle
* (c) a legendary Spacecraft with printed power/toughness
* 903.3a — any legendary card whose text says it can be your commander
*
* 903.3a is the awkward one: Scryfall and MTGJSON both strip that sentence
* from the card text they publish, so it cannot be read card by card from any
* accessible dataset. Wizards' own search indexes only the "can be your
* commander" wording and misses the rest, and its Partner filter omits every
* Partner planeswalker.
*
* The two classes that the stripped sentence would otherwise decide are
* therefore taken from the type line, which is correct for all of them:
*
* - Legendary Planeswalkers. Every one printed since the 2017 change that
* made planeswalkers legal commanders carries the ability, and the
* pre-2017 ones were reprinted with it.
* - Legendary Vehicles and Spacecraft, which 903.3b/c admit by type.
*
* What neither type line nor any dataset can supply is the handful of
* legendary artifacts, enchantments, lands and sorceries that lead a deck
* only because of 903.3a. That is a short, closed list, kept in
* DESIGNATED_COMMANDERS below. Add to it when a new set introduces one.
*/
const DESIGNATED_COMMANDERS = new Set([
'Agatha\'s Soul Cauldron',
'Gerrard\'s Hourglass Pendant',
'Instrument of the Bards',
'Lithoform Engine',
'Nyx Lotus',
'Ozolith, the Shattered Spire',
'The Book of Exalted Deeds',
'The Cauldron of Eternity',
'The Earth Crystal',
'The Mightstone and Weakstone',
"Urza's Sylex",
])
/** A Background is a legendary creature but can never lead a deck alone. */
function isCommanderEligible(card) {
const typeLine = card.type_line ?? ''
if (!/legendary/i.test(typeLine)) return false
if (/\bbackground\b/i.test(typeLine)) return false
if (/creature/i.test(typeLine) || /vehicle/i.test(typeLine) || /\bplaneswalker\b/i.test(typeLine)) return true
if (/\bspacecraft\b/i.test(typeLine)) return card.power != null && card.toughness != null
return (
DESIGNATED_COMMANDERS.has((card.name ?? '').split(' // ')[0]) ||
/can be your commander/i.test(card.oracle_text ?? '')
)
}
function project(card, commanderLegality) {
const typeLine = card.type_line ?? ''
const oracleText = card.oracle_text ?? ''
const name = (card.name ?? '').split(' // ')[0]
const formats = []
for (const [format, status] of Object.entries(card.legalities ?? {})) {
if (status === 'legal') formats.push(format)
}
const cmc = Number.isFinite(card.cmc) ? card.cmc : 0
// Scryfall models both of these as arrays; a colourless card has an empty
// one, which must not be bound to a TEXT column.
const identity = (card.color_identity ?? []).join('')
const colors = (card.colors ?? []).join('')
const commanderEligible = isCommanderEligible(card)
return [
card.id,
card.oracle_id ?? card.id,
name,
card.mana_cost ?? '',
cmc,
Math.min(cmc, 7),
typeLine,
oracleText,
identity,
colors,
card.rarity ?? '',
card.power ?? null,
card.toughness ?? null,
card.loyalty ?? null,
card.layout ?? '',
card.faces ? JSON.stringify(card.faces) : null,
card.released_at ?? '',
card.artist ?? '',
card.set_name ?? '',
card.set_code ?? '',
commanderLegality,
commanderEligible ? 1 : 0,
/\bpartner\b/i.test(oracleText) ? 1 : 0,
/^basic (snow )?land\b/i.test(typeLine.trim()) ? 1 : 0,
JSON.stringify(formats),
card.image_uris?.small ?? null,
card.image_uris?.normal ?? null,
card.image_uris?.large ?? null,
card.image_uris?.png ?? null,
card.image_uris?.art_crop ?? null,
card.image_uris?.board_crop ?? null,
]
}
main().catch((err) => {
console.error(err)
process.exitCode = 1
})