Skip to content

armgit / mocha

Public Imported from GitHub
0

☕️ Classic, reliable, trusted test framework for Node.js and the browser

9.7 KB
Raw Blame
/**
 * This script gathers metadata for active supporters of Mocha from Open Collective's
 * API by aggregating order ("donation") information.
 *
 * - Gathers logo/avatar images (they are always pngs)
 * - Gathers links
 * - Sorts by tier and total contributions
 * - Validates images
 * - Writes images to a temp dir
 *
 * We filter out spam based on the code below and these general guidelines:
 * - Not based on deceiving people
 *
 * While we try our best, we cannot guarantee quality in the list of sponsors.
 * Some non-spam may be blocked, and some spam may be allowed.
 * Please let us know in the usual places if you feel there is a mistake.
 *
 * @see https://docs.opencollective.com/help/contributing/development/api
 */

"use strict";

const { writeFile, mkdir, rm } = require("node:fs").promises;
const { resolve } = require("node:path");
const debug = require("debug")("mocha:docs:data:supporters");
const blocklist = new Set(require("./blocklist.json"));

/**
 * In addition to the blocklist, any account slug matching this regex will not
 * be displayed on the website.
 */
const BLOCKED_STRINGS =
  /(?:[ck]a[sz]ino|seo|slot|gambl(?:e|ing)|crypto|cheap|instagram|hacks|tiktok|likes|subscriber|boost|deposit|mushroom|bingo|broker|promotion|groundskeeping|links|review|refer|promocode|rabattkod|jämför|betting|reddit|hire|fortune|equity|download|marketing|comment|rank|scrapcar|lawyer|celeb|concrete|firestick|playground|betking)/i;

/**
 * Add a few Categories exposed by Open Collective to help moderation
 */
const BLOCKED_CATEGORIES = [
  "adult",
  "casino",
  "credit",
  "gambling",
  "seo",
  "writer",
  "review",
];

/**
 * The OC API endpoint
 */
const API_ENDPOINT = "https://api.opencollective.com/graphql/v2";

const SPONSOR_TIER = "sponsors";
const BACKER_TIER = "backers";

// if this percent of fetches completes, the build will pass
const PRODUCTION_SUCCESS_THRESHOLD = 0.9;

const SUPPORTER_IMAGE_PATH = resolve(__dirname, "../images/supporters");

const SUPPORTER_QUERY = `query account($limit: Int, $offset: Int, $slug: String) {
  account(slug: $slug) {
    orders(limit: $limit, offset: $offset, status: ACTIVE, filter: INCOMING) {
      limit
      offset
      totalCount
      nodes {
        fromAccount {
          id
          name
          slug
          website
          imgUrlMed: imageUrl(height:64)
          imgUrlSmall: imageUrl(height:32)
          type
          categories
        }
        tier { slug }
        totalDonations { value }
        createdAt
      }
    }
  }
}`;

const GRAPHQL_PAGE_SIZE = 1000;

const invalidSupporters = [];

const nodeToSupporter = (node) => ({
  id: node.fromAccount.id,
  name: node.fromAccount.name,
  slug: node.fromAccount.slug,
  website: node.fromAccount.website,
  imgUrlMed: node.fromAccount.imgUrlMed,
  imgUrlSmall: node.fromAccount.imgUrlSmall,
  type: node.fromAccount.type,
  categories: node.fromAccount.categories,
  tier: (node.tier && node.tier.slug) || BACKER_TIER,
  totalDonations: node.totalDonations.value * 100,
  firstDonation: node.createdAt,
});

const fetchImage = process.env.MOCHA_DOCS_SKIP_IMAGE_DOWNLOAD
  ? async (supporter) => {
      invalidSupporters.push(supporter);
    }
  : async (supporter) => {
      try {
        const { avatar: url } = supporter;
        const response = await fetch(url, {
          signal: AbortSignal.timeout(30000),
        });
        if (response.headers.get("content-type")?.startsWith("text/html")) {
          throw new TypeError(
            "received html and expected a png; outage likely",
          );
        }
        const imageBuf = Buffer.from(await response.arrayBuffer());
        debug("fetched %s", url);
        const filePath = resolve(SUPPORTER_IMAGE_PATH, supporter.id + ".png");
        await writeFile(filePath, imageBuf);
        debug("wrote %s", filePath);
      } catch (err) {
        console.error(
          `failed to load ${supporter.avatar}; will discard ${supporter.tier} "${supporter.name} (${supporter.slug}). reason:\n`,
          err,
        );
        invalidSupporters.push(supporter);
      }
    };

/**
 * Retrieves donation data from OC
 *
 * Handles pagination
 * @param {string} slug - Collective slug to get donation data from
 * @returns {Promise<Object[]>} Array of raw donation data
 */
const getAllOrders = async (slug = "mochajs") => {
  let allOrders = [];
  const variables = { limit: GRAPHQL_PAGE_SIZE, offset: 0, slug };

  // Handling pagination if necessary (2 pages for ~1400 results in May 2019)
  while (true) {
    const response = await fetch(API_ENDPOINT, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({
        query: SUPPORTER_QUERY,
        variables: variables,
      }),
    });
    const result = await response.json();
    const orders = result.data.account.orders.nodes;
    allOrders = [...allOrders, ...orders];
    variables.offset += GRAPHQL_PAGE_SIZE;
    if (orders.length < GRAPHQL_PAGE_SIZE) {
      debug("retrieved %d orders", allOrders.length);
      return allOrders;
    } else {
      debug(
        "loading page %d of orders...",
        Math.floor(variables.offset / GRAPHQL_PAGE_SIZE),
      );
    }
  }
};

const isAllowed = ({ name, slug, website, categories }) => {
  const blockReasons = [];
  if (blocklist.has(slug)) blockReasons.push("slug in blocklist");
  if (BLOCKED_STRINGS.test(name)) blockReasons.push("name blocked");
  if (BLOCKED_STRINGS.test(slug)) blockReasons.push("slug blocked");
  if (BLOCKED_STRINGS.test(website)) blockReasons.push("website blocked");
  if (categories.some((category) => BLOCKED_CATEGORIES.includes(category)))
    blockReasons.push("category blocked");

  const allowed = blockReasons.length === 0;

  if (!allowed) {
    debug("blocking %o", { categories, name, slug, website });
    debug(`reason(s): ${blockReasons}`);
  } else {
    debug("allowing %o", { categories, name, slug, website });
  }
  return allowed;
};

const getSupporters = async () => {
  const orders = await getAllOrders();
  // Deduplicating supporters with multiple orders
  const uniqueSupporters = new Map();

  const supporters = orders
    // turn raw query result into a better data structure
    .map(nodeToSupporter)
    // aggregate total $ donated by unique supporter (using slug)
    .reduce((supporters, supporter) => {
      if (uniqueSupporters.has(supporter.slug)) {
        uniqueSupporters.get(supporter.slug).totalDonations +=
          supporter.totalDonations;
        return supporters;
      }
      uniqueSupporters.set(supporter.slug, supporter);
      return [...supporters, supporter];
    }, [])
    // discard spammy supporters
    .filter(isAllowed)
    // sort by total $ donated, descending
    .sort((a, b) => b.totalDonations - a.totalDonations)
    // determine which url to use depending on tier
    .reduce(
      (supporters, supporter) => {
        if (supporter.tier === BACKER_TIER) {
          if (supporter.name !== "anonymous") {
            supporters[BACKER_TIER] = [
              ...supporters[BACKER_TIER],
              {
                ...supporter,
                avatar: encodeURI(supporter.imgUrlSmall),
              },
            ];
          }
        } else {
          supporters[SPONSOR_TIER] = [
            ...supporters[SPONSOR_TIER],
            {
              ...supporter,
              avatar: encodeURI(supporter.imgUrlMed),
            },
          ];
        }
        return supporters;
      },
      {
        [SPONSOR_TIER]: [],
        [BACKER_TIER]: [],
      },
    );

  await rm(SUPPORTER_IMAGE_PATH, { recursive: true, force: true });
  debug("blasted %s", SUPPORTER_IMAGE_PATH);
  await mkdir(SUPPORTER_IMAGE_PATH, { recursive: true });
  debug("created %s", SUPPORTER_IMAGE_PATH);

  // Fetch images for sponsors and save their image dimensions
  await Promise.all([
    ...supporters[SPONSOR_TIER].map(fetchImage),
    ...supporters[BACKER_TIER].map(fetchImage),
  ]);
  debug("fetched images");

  invalidSupporters.forEach((supporter) => {
    supporters[supporter.tier].splice(
      supporters[supporter.tier].indexOf(supporter),
      1,
    );
  });
  debug("tossed out invalid supporters");

  const backerCount = supporters[BACKER_TIER].length;
  const sponsorCount = supporters[SPONSOR_TIER].length;
  const totalSupportersCount = backerCount + sponsorCount;
  const successRate = 1 - invalidSupporters.length / totalSupportersCount;

  debug(
    "found %d valid backers and %d valid sponsors (%d total; %d invalid; %d blocked)",
    backerCount,
    sponsorCount,
    totalSupportersCount,
    invalidSupporters.length,
    uniqueSupporters.size - totalSupportersCount,
  );

  if (successRate < PRODUCTION_SUCCESS_THRESHOLD) {
    if (process.env.NETLIFY && process.env.CONTEXT !== "deploy-preview") {
      throw new Error(
        `Failed to meet success threshold ${
          PRODUCTION_SUCCESS_THRESHOLD * 100
        }% (was ${
          successRate * 100
        }%) for a production deployment; refusing to deploy`,
      );
    } else {
      console.warn(
        `WARNING: Success rate of ${
          successRate * 100
        }% fails to meet production threshold of ${
          PRODUCTION_SUCCESS_THRESHOLD * 100
        }%; would fail a production deployment!`,
      );
    }
  }
  debug("supporter image pull completed");

  // TODO: we'll sunset the classic docs and only have docs.
  // At that point we'll have supporters.js only used for writing files.
  if (process.argv.includes("--write-supporters-json")) {
    await mkdir("src/content/data", { recursive: true });
    await writeFile(
      "src/content/data/supporters.json",
      JSON.stringify(supporters, null, 4),
    );
  }
  return supporters;
};

module.exports = getSupporters;

if (require.main === module) {
  require("debug").enable("mocha:docs:data:supporters");
  process.on("unhandledRejection", (err) => {
    throw err;
  });
  getSupporters();
}