bigtech-sucks/packages/core/db/build.mjs

#!/usr/bin/env node
/**
 * bigtech.sucks — database build
 *
 * The database is the source of truth. This script:
 *   1. rebuilds bigtech.db from schema + seed
 *   2. absorbs the legacy hand-authored outlaw roster as UNASSESSED products
 *   3. applies every db/seed/*.sql in name order
 *   4. verifies the result against the instrument's own rules
 *   5. exports a generated module the website imports
 *
 * Run: npm run db:build
 * Verify only: npm run db:verify
 */
import { DatabaseSync } from 'node:sqlite';
import { readFileSync, writeFileSync, readdirSync, mkdirSync, existsSync, unlinkSync } from 'node:fs';
import { createHash } from 'node:crypto';
import { effectiveLevel, capFor } from './attest/validate.mjs';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';

const HERE = dirname(fileURLToPath(import.meta.url));
const CORE = join(HERE, '..');
const DB_PATH = join(HERE, 'bigtech.db');
const SEED_DIR = join(HERE, 'seed');
const GEN_DIR = join(CORE, 'src', 'data', 'generated');

const GRADES = (n) => (n >= 90 ? 'A' : n >= 75 ? 'B' : n >= 60 ? 'C' : n >= 40 ? 'D' : 'F');

// ---------------------------------------------------------------- vendor map
// Outlaw id -> vendor. Everything here is checkable public record.
const VENDORS = {
  google:            ['Google LLC', 'private', 'US', 'advertising'],
  microsoft:         ['Microsoft Corporation', 'public', 'US', 'mixed'],
  apple:             ['Apple Inc.', 'public', 'US', 'hardware'],
  meta:              ['Meta Platforms', 'public', 'US', 'advertising'],
  amazon:            ['Amazon.com, Inc.', 'public', 'US', 'mixed'],
  salesforce:        ['Salesforce, Inc.', 'public', 'US', 'subscription'],
  atlassian:         ['Atlassian Corporation', 'public', 'AU', 'subscription'],
  zoom:              ['Zoom Communications', 'public', 'US', 'subscription'],
  discord:           ['Discord Inc.', 'private', 'US', 'subscription'],
  otter:             ['Otter.ai', 'private', 'US', 'subscription'],
  notion:            ['Notion Labs', 'private', 'US', 'subscription'],
  evernote:          ['Evernote (Bending Spoons)', 'private', 'IT', 'subscription'],
  obsidian:          ['Dynalist Inc. (Obsidian)', 'private', 'CA', 'license'],
  openai:            ['OpenAI', 'private', 'US', 'mixed'],
  lastpass:          ['LastPass (GoTo)', 'private', 'US', 'subscription'],
  '1password':       ['AgileBits Inc. (1Password)', 'private', 'CA', 'subscription'],
  proton:            ['Proton AG', 'private', 'CH', 'subscription'],
  dropbox:           ['Dropbox, Inc.', 'public', 'US', 'subscription'],
  'x-corp':          ['X Corp.', 'private', 'US', 'advertising'],
  intuit:            ['Intuit Inc.', 'public', 'US', 'subscription'],
  plaid:             ['Plaid Inc.', 'private', 'US', 'mixed'],
  asana:             ['Asana, Inc.', 'public', 'US', 'subscription'],
  '23andme':         ['23andMe Holding Co.', 'public', 'US', 'mixed'],
  vercel:            ['Vercel Inc.', 'private', 'US', 'subscription'],
  'nord-security':   ['Nord Security', 'private', 'PA', 'subscription'],
  netflix:           ['Netflix, Inc.', 'public', 'US', 'subscription'],
  'nextcloud-gmbh':  ['Nextcloud GmbH', 'private', 'DE', 'subscription'],
  'signal-foundation': ['Signal Technology Foundation', 'nonprofit', 'US', 'none'],
  'kino-ai':         ['Kino AI', 'private', 'US', 'subscription'],
  // discovered products land here until a human or agent resolves the owner
  unknown:           ['Unknown vendor', 'unknown', 'unknown', 'unknown'],
};

const OUTLAW_VENDOR = {
  gmail: 'google', outlook: 'microsoft', protonmail: 'proton',
  google_drive: 'google', dropbox: 'dropbox', onedrive: 'microsoft',
  google_workspace: 'google', microsoft_365: 'microsoft',
  slack: 'salesforce', discord: 'discord', zoom: 'zoom', otter: 'otter',
  teams: 'microsoft', notion: 'notion', evernote: 'evernote',
  obsidian_sync: 'obsidian', whisper_api: 'openai', lastpass: 'lastpass',
  '1password': '1password', google_calendar: 'google',
  google_photos: 'google', icloud_photos: 'apple', chrome: 'google',
  chatgpt: 'openai', gemini: 'google', twitter: 'x-corp',
  facebook: 'meta', mint: 'intuit', plaid: 'plaid',
  salesforce: 'salesforce', asana: 'asana', trello: 'atlassian',
  '23andme': '23andme', fitbit: 'google', ring: 'amazon', nest: 'google',
  github: 'microsoft', vercel: 'vercel', nordvpn: 'nord-security',
  netflix: 'netflix', windows: 'microsoft', android: 'google',
  nextcloud: 'nextcloud-gmbh',
};

const ID_OVERRIDES = { '1password': 'onepassword', '23andme': 'twentythreeandme' };
const slug = (s) => ID_OVERRIDES[s] ?? s.replace(/[^a-z0-9]+/gi, '-').toLowerCase();

// ---------------------------------------------------------------- build
async function build({ fresh = false } = {}) {
  // Never wipe by default: the database authors itself, and a rebuild that
  // deletes discovered products would throw away the roster. --fresh is explicit.
  if (fresh && existsSync(DB_PATH)) unlinkSync(DB_PATH);
  const db = new DatabaseSync(DB_PATH);
  db.exec('PRAGMA foreign_keys = ON');
  db.exec(readFileSync(join(HERE, 'schema.sql'), 'utf8'));
  migrate(db);
  // second pass: migrate drops the views so this recreates them with the current
  // definitions. Views cannot be altered, only replaced.
  db.exec(readFileSync(join(HERE, 'schema.sql'), 'utf8'));

  // vendors
  const insVendor = db.prepare(`INSERT OR REPLACE INTO vendor
    (id, name, legal_entity, entity_type, jurisdiction, business_model) VALUES (?,?,?,?,?,?)`);
  for (const [id, [name, type, juris, model]] of Object.entries(VENDORS)) {
    insVendor.run(id, name.replace(/\s*\(.*\)$/, ''), name, type, juris, model);
  }

  // categories and legacy roster, imported from the built core package so the
  // database is guaranteed to match what the site shipped with.
  const { categories } = await import(join(CORE, 'dist', 'data', 'categories.js'));
  const { outlaws } = await import(join(CORE, 'dist', 'data', 'outlaws.js'));

  const insCat = db.prepare(`INSERT OR REPLACE INTO category (id,label,sensitivity,nextcloud) VALUES (?,?,?,?)`);
  for (const c of categories) insCat.run(c.id, c.label, c.sensitivity, c.nc);

  const insProd = db.prepare(`INSERT OR IGNORE INTO product
    (id,vendor_id,name,category_id,platform,sovereign,boss,art_slot,monthly_cost,notes)
    VALUES (?,?,?,?,?,?,?,?,?,?)`);
  let migrated = 0;
  for (const o of outlaws) {
    const vendorId = OUTLAW_VENDOR[o.id];
    if (!vendorId) { console.warn(`  ! no vendor mapped for outlaw '${o.id}'`); continue; }
    insProd.run(slug(o.id), vendorId, o.name, o.category, 'web', o.sovereign ? 1 : 0,
      o.boss ? 1 : 0, o.faceArtSlot, o.monthlyCost ?? null,
      `Legacy editorial placeholder. Not yet assessed. Original policy=${o.policy} infra=${o.infra}.`);
    migrated++;
  }

  // seeds
  const seeds = readdirSync(SEED_DIR).filter((f) => f.endsWith('.sql')).sort();
  for (const f of seeds) db.exec(readFileSync(join(SEED_DIR, f), 'utf8'));

  db.close();
  return { migrated, seeds };
}

// ---------------------------------------------------------------- migrate
// The database authors itself, so it must survive its own schema changes.
// Never drop and rebuild: add the column, drop the views, let schema.sql recreate.
function migrate(db) {
  const cols = (t) => db.prepare(`PRAGMA table_info(${t})`).all().map((c) => c.name);

  if (!cols('product').includes('priority')) {
    db.exec('ALTER TABLE product ADD COLUMN priority INTEGER NOT NULL DEFAULT 0');
  }
  if (!cols('assessment').includes('source_available')) {
    db.exec('ALTER TABLE assessment ADD COLUMN source_available INTEGER NOT NULL DEFAULT 1');
  }
  for (const v of ['v_backlog', 'v_coverage', 'v_product_score', 'v_requested']) {
    db.exec(`DROP VIEW IF EXISTS ${v}`);
  }
}

// ---------------------------------------------------------------- verify
function verify() {
  const db = new DatabaseSync(DB_PATH, { readOnly: true });
  const fails = [];
  const ok = (cond, msg) => { if (!cond) fails.push(msg); };

  const dims = db.prepare('SELECT id, weight FROM dimension').all();
  const weightSum = dims.reduce((a, d) => a + d.weight, 0);
  ok(weightSum === 100, `dimension weights sum to ${weightSum}, expected 100`);

  const assessments = db.prepare('SELECT * FROM assessment').all();
  ok(assessments.length > 0, 'no assessments present');

  for (const a of assessments) {
    const scores = db.prepare('SELECT s.*, d.weight FROM score s JOIN dimension d ON d.id = s.dimension_id WHERE s.assessment_id = ?').all(a.id);
    ok(scores.length === dims.length, `${a.id}: ${scores.length} scores, expected ${dims.length}`);
    const recomputed = scores.reduce((acc, s) => acc + s.value * s.weight, 0) / 10;
    ok(Math.abs(recomputed - a.weighted_total) < 0.05,
       `${a.id}: stored total ${a.weighted_total} but recomputed ${recomputed.toFixed(2)}`);
    ok(GRADES(a.weighted_total) === a.grade,
       `${a.id}: grade ${a.grade} does not match band for ${a.weighted_total}`);
    for (const s of scores) ok(!!s.rationale && s.rationale.length > 20,
       `${a.id}/${s.dimension_id}: score has no written rationale`);

    const t12 = db.prepare('SELECT COUNT(*) AS n FROM evidence WHERE assessment_id = ? AND tier <= 2').get(a.id).n;
    ok(t12 >= 3, `${a.id}: only ${t12} tier-1/2 evidence rows, need 3`);

    const t4 = db.prepare('SELECT COUNT(*) AS n FROM evidence WHERE assessment_id = ? AND tier = 4 AND dimension_id IS NOT NULL').get(a.id).n;
    ok(t4 === 0, `${a.id}: ${t4} tier-4 claims attached to a scored dimension`);

    const lowest = scores.reduce((m, s) => (s.value < m.value ? s : m), scores[0]);
    ok(lowest.dimension_id === a.weakest_dimension,
       `${a.id}: weakest_dimension is ${a.weakest_dimension} but lowest score is ${lowest.dimension_id}`);
  }

  // The instrument's caps must be real, not documentation.
  for (const a of assessments) {
    const v = db.prepare("SELECT value FROM score WHERE assessment_id = ? AND dimension_id = 'verifiability'").get(a.id)?.value;
    if (a.reproducible_build === 0) ok(v <= 3, `${a.id}: no reproducible build but verifiability ${v} > 3`);
    if (a.reproducible_build === 1) ok(v <= 6, `${a.id}: experimental reproducible build but verifiability ${v} > 6`);

    // Closed source starts at level 0 and climbs a public ladder. The ceiling is
    // not a punishment, it is the rung you are standing on.
    //
    //   L0 unattested   -> 3    (we have nothing but your word)
    //   L1 declared     -> 5    (SBOM, signed hash, published destination allowlist)
    //   L2 observed     -> 7    (someone other than you watched this exact binary)
    //   L3 reproducible -> 9    (anyone can rebuild it and get the same bytes)
    //   L4 contested    -> 10   (someone tried to break it and published the attempt)
    //
    // A court filing is independent evidence and needs no vendor cooperation, so
    // it lifts the ceiling entirely.
    if (a.source_available === 0) {
      const att = db.prepare(`
        SELECT * FROM attestation
        WHERE product_id = ? AND version = ?
        ORDER BY issued_on DESC LIMIT 1`).get(a.product_id, a.version);
      const level = att ? effectiveLevel(att) : 0;
      const dims = db.prepare('SELECT * FROM score WHERE assessment_id = ?').all(a.id);
      for (const s of dims) {
        const filed = db.prepare(`
          SELECT COUNT(*) AS n FROM evidence
          WHERE assessment_id = ? AND dimension_id = ? AND tier <= 2 AND kind = 'filing'`)
          .get(a.id, s.dimension_id).n;
        const ceiling = filed > 0 ? 10 : capFor(s.dimension_id, level);
        ok(s.value <= ceiling,
           `${a.id}/${s.dimension_id}: scored ${s.value} but the ceiling at attestation level ${level} is ${ceiling}`);
      }
    }
  }

  const scores = db.prepare('SELECT * FROM v_product_score').all();
  for (const s of scores) {
    if (s.weighted_total === null) continue;
    ok(s.policy >= 0 && s.policy <= 100, `${s.product}: policy ${s.policy} out of range`);
    ok(s.infra >= 0 && s.infra <= 100, `${s.product}: infra ${s.infra} out of range`);
  }

  const cov = db.prepare('SELECT * FROM v_coverage').all();
  const assessed = cov.filter((c) => c.status !== 'unassessed').length;
  const total = cov.length;

  db.close();
  return { fails, scores, cov, assessed, total };
}

// ---------------------------------------------------------------- export
function exportSite() {
  const db = new DatabaseSync(DB_PATH, { readOnly: true });
  mkdirSync(GEN_DIR, { recursive: true });

  const products = db.prepare(`
    SELECT p.id, p.name, p.platform, p.category_id, p.vendor_id, p.sovereign, p.boss, p.art_slot,
           p.monthly_cost, v.name AS vendor, v.entity_type, v.jurisdiction, v.business_model,
           s.weighted_total, s.grade, s.policy, s.infra, s.weakest_dimension, s.headline, s.assessed_on
    FROM product p
    JOIN vendor v ON v.id = p.vendor_id
    LEFT JOIN v_product_score s ON s.product_id = p.id
    ORDER BY p.category_id, p.name`).all();

  const assessments = db.prepare(`
    SELECT a.id, a.product_id, a.version, a.source_commit, a.assessed_on, a.method_version,
           a.reproducible_build, a.weighted_total, a.grade, a.weakest_dimension, a.headline, a.not_tested
    FROM assessment a ORDER BY a.assessed_on DESC`).all();

  const scores = db.prepare(`
    SELECT s.assessment_id, s.dimension_id, s.value, s.rationale, d.label, d.weight
    FROM score s JOIN dimension d ON d.id = s.dimension_id
    ORDER BY d.weight DESC`).all();

  const evidence = db.prepare(`
    SELECT id, assessment_id, dimension_id, claim, tier, kind, artifact, sha256, observed_at
    FROM evidence ORDER BY assessment_id, tier, id`).all();

  const findings = db.prepare(`
    SELECT id, assessment_id, severity, title, detail, status, cve
    FROM finding ORDER BY assessment_id,
      CASE severity WHEN 'critical' THEN 0 WHEN 'high' THEN 1 WHEN 'medium' THEN 2
                    WHEN 'low' THEN 3 ELSE 4 END`).all();

  const dimensions = db.prepare('SELECT * FROM dimension ORDER BY weight DESC').all();
  db.close();

  const payload = {
    generated_at: new Date().toISOString(),
    method_version: 'btr-0.1',
    dimensions, products, assessments, scores, evidence, findings,
  };

  // The bytes we hash are the bytes we serve. The oath is bound to this digest,
  // so an oath page that does not match the data is detectable by a stranger.
  const catalogBytes = JSON.stringify(payload, null, 2);
  writeFileSync(join(GEN_DIR, 'catalog.json'), catalogBytes);

  const PUB = join(CORE, '..', 'web', 'public');
  mkdirSync(PUB, { recursive: true });
  writeFileSync(join(PUB, 'catalog.json'), catalogBytes);

  const digest = createHash('sha256').update(catalogBytes).digest('hex');
  const oath = {
    digest,
    algorithm: 'sha256',
    file: 'catalog.json',
    byte_length: Buffer.byteLength(catalogBytes),
    rebuilt_at: new Date().toISOString(),
    method_version: payload.method_version,
    products: products.length,
    assessed: products.filter((p) => p.weighted_total !== null).length,
  };
  writeFileSync(join(PUB, 'oath.json'), JSON.stringify(oath, null, 2));

  const ts = `// GENERATED FILE - do not edit by hand.
// Source of truth: packages/core/db/bigtech.db
// Rebuild: npm run db:build

export interface CatalogProduct {
  id: string; name: string; platform: string; category_id: string; vendor_id: string;
  vendor: string; entity_type: string; jurisdiction: string; business_model: string;
  sovereign: number; boss: number; art_slot: string | null; monthly_cost: number | null;
  weighted_total: number | null; grade: string | null; policy: number | null;
  infra: number | null; weakest_dimension: string | null; headline: string | null;
  assessed_on: string | null;
}

export interface CatalogAssessment {
  id: string; product_id: string; version: string; source_commit: string | null;
  assessed_on: string; method_version: string; reproducible_build: number;
  weighted_total: number; grade: string; weakest_dimension: string; headline: string;
  not_tested: string | null;
}

export interface CatalogScore {
  assessment_id: string; dimension_id: string; value: number;
  rationale: string; label: string; weight: number;
}

export interface CatalogEvidence {
  id: number; assessment_id: string; dimension_id: string | null; claim: string;
  tier: number; kind: string | null; artifact: string | null;
  sha256: string | null; observed_at: string | null;
}

export interface CatalogFinding {
  id: number; assessment_id: string; severity: string; title: string;
  detail: string | null; status: string; cve: string | null;
}

export interface CatalogDimension {
  id: string; label: string; weight: number; anchor_10: string; anchor_0: string;
}

export const catalog = ${JSON.stringify(payload, null, 2)} as {
  generated_at: string;
  method_version: string;
  dimensions: CatalogDimension[];
  products: CatalogProduct[];
  assessments: CatalogAssessment[];
  scores: CatalogScore[];
  evidence: CatalogEvidence[];
  findings: CatalogFinding[];
};

/** Products with a real assessment. Everything else is a placeholder. */
export const assessed = catalog.products.filter((p) => p.weighted_total !== null);

/** The full roster the game renders, measured where we have measured it. */
export const products = catalog.products;

export function productById(id: string): CatalogProduct | undefined {
  return catalog.products.find((p) => p.id === id);
}
`;
  writeFileSync(join(GEN_DIR, 'catalog.ts'), ts);
  return payload;
}

// ---------------------------------------------------------------- run
const mode = process.argv[2] ?? 'all';
if (mode === 'verify') {
  const r = verify();
  report(r);
} else {
  const fresh = process.argv.includes('--fresh');
  const { migrated, seeds } = await build({ fresh });
  console.log(`built ${DB_PATH}`);
  console.log(`  legacy roster absorbed: ${migrated} products (unassessed)`);
  console.log(`  seeds applied: ${seeds.join(', ')}`);
  const r = verify();
  const payload = exportSite();
  console.log(`  exported: src/data/generated/catalog.{ts,json} (${payload.products.length} products)`);
  report(r);
}

function report({ fails, cov, assessed, total }) {
  console.log('');
  console.log('VERIFY');
  if (fails.length === 0) console.log('  all instrument checks passed');
  else for (const f of fails) console.log(`  FAIL ${f}`);
  console.log(`  coverage: ${assessed} assessed / ${total} products (${((assessed / total) * 100).toFixed(1)}%)`);
  console.log('');
  console.log('SCORED');
  for (const c of cov.filter((x) => x.status !== 'unassessed')) {
    console.log(`  ${c.name} [${c.platform}]  ${c.weighted_total}/100  grade ${c.status}`);
  }
  if (fails.length) process.exitCode = 1;
}