d433a0e75e
The roster writes itself. The scores do not. - db/ingest/sources.mjs: GitHub and Flathub adapters. GitHub is ranked by stars (usage); Flathub is a curated store of end-user Linux apps, so relevance is free there and has to be earned on GitHub. - db/ingest/run.mjs: discovers into the roster as unassessed products, never as scores. Every discovery leaves a receipt (source, url, usage metric, timestamp) so a reader can see where a product came from and why it ranks. - LooksLikeUserSoftware gate: topic:messaging sorts RabbitMQ to the top because brokers are highly starred and are not things people install. - build.mjs no longer wipes the database. The DB authors itself, so a rebuild would have thrown the roster away. --fresh is now explicit. - v_backlog view: what to assess next, ordered by measured usage. - packages/web/public/oath.html: the Ranch Oath, with an ASCII Bible crest, bound to a SHA-256 of catalog.json. The page recomputes the digest in the browser and reports VERIFIED or BROKEN. An oath you cannot break is decoration. Roster now 122 products, 1 assessed. All instrument checks pass.
97 lines
3.9 KiB
JavaScript
97 lines
3.9 KiB
JavaScript
/**
|
|
* Discovery adapters.
|
|
*
|
|
* Each adapter returns a normalised shape:
|
|
* { name, platform, sourceUrl, usage, usageMetric, raw }
|
|
*
|
|
* Rules for an adapter:
|
|
* - it discovers SOFTWARE, it never scores it
|
|
* - it records where it found the thing and how much it is used
|
|
* - it fails loudly, never silently returning an empty list on a blocked fetch
|
|
*
|
|
* Sources currently live:
|
|
* github open API, no key needed, 60 requests/hour unauthenticated
|
|
* flathub open API, POST /api/v2/search, no key
|
|
*
|
|
* Sources deliberately not automated:
|
|
* alternativeto Cloudflare-protected. Readable through a real browser engine
|
|
* (verified 2026-10-29) but their robots.txt asks ai-train=no.
|
|
* Treated as a demand signal for manual/periodic import, not a
|
|
* crawler target. Do not add a scraper here without a decision.
|
|
*/
|
|
|
|
const UA = 'bigtech-sucks-ingest/0.1 (+https://bigtech.sucks; assessment roster)';
|
|
|
|
async function getJson(url, init = {}) {
|
|
const res = await fetch(url, {
|
|
...init,
|
|
headers: { 'User-Agent': UA, Accept: 'application/json', ...(init.headers || {}) },
|
|
});
|
|
if (!res.ok) throw new Error(`${url} -> HTTP ${res.status}`);
|
|
return res.json();
|
|
}
|
|
|
|
// ------------------------------------------------------------------ relevance
|
|
// Usage rank alone is not a roster. `topic:messaging` sorts RabbitMQ to the top,
|
|
// because a message broker is highly starred and not a thing a human installs.
|
|
// A candidate must look like software a person uses, or it is dropped and the
|
|
// reason is recorded.
|
|
const NOT_AN_APP =
|
|
/\b(broker|queue|mqtt|amqp|kafka|nats|pulsar|rocketmq|emqx|nsq|rabbitmq|database|sdk|library|libraries|framework|protocol|server|infrastructure|plugin|extension|binding|middleware|gateway|proxy|driver|api client|spec|specification|awesome list)\b/i;
|
|
const APP_HINT =
|
|
/\b(app|application|client|desktop|chat|messenger|messaging|mobile|gui|editor|browser|vault|manager|suite)\b/i;
|
|
|
|
export function looksLikeUserSoftware(row) {
|
|
const text = `${row.name} ${row.raw?.description ?? ''}`;
|
|
if (NOT_AN_APP.test(text)) return false;
|
|
if (row.raw?.homepage) return true;
|
|
return APP_HINT.test(text);
|
|
}
|
|
|
|
// ------------------------------------------------------------------ GitHub
|
|
export async function githubByTopic(topic, { minStars = 500, limit = 10 } = {}) {
|
|
const q = encodeURIComponent(`topic:${topic} stars:>=${minStars} archived:false`);
|
|
const data = await getJson(
|
|
`https://api.github.com/search/repositories?q=${q}&sort=stars&order=desc&per_page=${limit}`,
|
|
);
|
|
return (data.items || []).map((r) => ({
|
|
name: r.name.replace(/[-_]/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase()),
|
|
platform: 'source',
|
|
sourceUrl: r.html_url,
|
|
usage: r.stargazers_count ?? 0,
|
|
usageMetric: 'github_stars',
|
|
raw: {
|
|
full_name: r.full_name,
|
|
stars: r.stargazers_count,
|
|
forks: r.forks_count,
|
|
license: r.license?.spdx_id ?? null,
|
|
description: r.description,
|
|
homepage: r.homepage || null,
|
|
language: r.language,
|
|
},
|
|
}));
|
|
}
|
|
|
|
// ------------------------------------------------------------------ Flathub
|
|
export async function flathubSearch(query, { limit = 10 } = {}) {
|
|
const data = await getJson('https://flathub.org/api/v2/search', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ query, filters: [] }),
|
|
});
|
|
return (data.hits || []).slice(0, limit).map((h) => ({
|
|
name: h.name,
|
|
platform: 'linux-flatpak',
|
|
sourceUrl: h.app_id ? `https://flathub.org/apps/${h.app_id}` : null,
|
|
// Flathub does not publish install counts in this endpoint. Presence in the
|
|
// distro store is a weak but real usage signal, scored low on purpose.
|
|
usage: 100,
|
|
usageMetric: 'flathub_presence',
|
|
raw: { app_id: h.app_id, summary: h.summary, developer: h.developer_name ?? null },
|
|
}));
|
|
}
|
|
|
|
export const ADAPTERS = {
|
|
github: (topic) => githubByTopic(topic),
|
|
flathub: (query) => flathubSearch(query),
|
|
};
|