bigtech-sucks/packages/core/db/ingest/sources.mjs

/**
 * Discovery adapters.
 *
 * Each adapter returns a normalised shape:
 *   { name, platform, sourceUrl, usage, usageMetric, raw }
 *
 * Rules for an adapter:
 *   - it discovers SOFTWARE, it never scores it
 *   - it records where it found the thing and how much it is used
 *   - it fails loudly, never silently returning an empty list on a blocked fetch
 *
 * Sources currently live:
 *   github   open API, no key needed, 60 requests/hour unauthenticated
 *   flathub  open API, POST /api/v2/search, no key
 *
 * Sources deliberately not automated:
 *   alternativeto  Cloudflare-protected. Readable through a real browser engine
 *                  (verified 2026-10-29) but their robots.txt asks ai-train=no.
 *                  Treated as a demand signal for manual/periodic import, not a
 *                  crawler target. Do not add a scraper here without a decision.
 */

const UA = 'bigtech-sucks-ingest/0.1 (+https://bigtech.sucks; assessment roster)';

async function getJson(url, init = {}) {
  const res = await fetch(url, {
    ...init,
    headers: { 'User-Agent': UA, Accept: 'application/json', ...(init.headers || {}) },
  });
  if (!res.ok) throw new Error(`${url} -> HTTP ${res.status}`);
  return res.json();
}

// ------------------------------------------------------------------ relevance
// Usage rank alone is not a roster. `topic:messaging` sorts RabbitMQ to the top,
// because a message broker is highly starred and not a thing a human installs.
// A candidate must look like software a person uses, or it is dropped and the
// reason is recorded.
const NOT_AN_APP =
  /\b(broker|queue|mqtt|amqp|kafka|nats|pulsar|rocketmq|emqx|nsq|rabbitmq|database|sdk|library|libraries|framework|protocol|server|infrastructure|plugin|extension|binding|middleware|gateway|proxy|driver|api client|spec|specification|awesome list)\b/i;
const APP_HINT =
  /\b(app|application|client|desktop|chat|messenger|messaging|mobile|gui|editor|browser|vault|manager|suite)\b/i;

export function looksLikeUserSoftware(row) {
  const text = `${row.name} ${row.raw?.description ?? ''}`;
  if (NOT_AN_APP.test(text)) return false;
  if (row.raw?.homepage) return true;
  return APP_HINT.test(text);
}

// ------------------------------------------------------------------ GitHub
export async function githubByTopic(topic, { minStars = 500, limit = 10 } = {}) {
  const q = encodeURIComponent(`topic:${topic} stars:>=${minStars} archived:false`);
  const data = await getJson(
    `https://api.github.com/search/repositories?q=${q}&sort=stars&order=desc&per_page=${limit}`,
  );
  return (data.items || []).map((r) => ({
    name: r.name.replace(/[-_]/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase()),
    platform: 'source',
    sourceUrl: r.html_url,
    usage: r.stargazers_count ?? 0,
    usageMetric: 'github_stars',
    raw: {
      full_name: r.full_name,
      stars: r.stargazers_count,
      forks: r.forks_count,
      license: r.license?.spdx_id ?? null,
      description: r.description,
      homepage: r.homepage || null,
      language: r.language,
    },
  }));
}

// ------------------------------------------------------------------ Flathub
export async function flathubSearch(query, { limit = 10 } = {}) {
  const data = await getJson('https://flathub.org/api/v2/search', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ query, filters: [] }),
  });
  return (data.hits || []).slice(0, limit).map((h) => ({
    name: h.name,
    platform: 'linux-flatpak',
    sourceUrl: h.app_id ? `https://flathub.org/apps/${h.app_id}` : null,
    // Flathub does not publish install counts in this endpoint. Presence in the
    // distro store is a weak but real usage signal, scored low on purpose.
    usage: 100,
    usageMetric: 'flathub_presence',
    raw: { app_id: h.app_id, summary: h.summary, developer: h.developer_name ?? null },
  }));
}

export const ADAPTERS = {
  github: (topic) => githubByTopic(topic),
  flathub: (query) => flathubSearch(query),
};