function condenseData()

in netlify/functions/store-ping-data-background/index.ts [17:68]


function condenseData(data: Ping[]): Pings {
	const fieldStringLookup = new Map<IndexedStringPingField, StringLookup>();
	const output: Pings = emptyPings(() => { return { strings: [], values: [] } });

	function getstr(field: IndexedStringPingField, s: string | null): StringIndex {
		if (!fieldStringLookup.has(field)) {
			fieldStringLookup.set(field, new Map<string | null, number>());
		}
		const stringLookup = fieldStringLookup.get(field)!;

		if (!stringLookup.has(s)) {
			stringLookup.set(s, output[field].strings.length);
			// We've verified that non-null fields will have only non-null
			// values (otherwise the ping is dropped), so we can safely push
			// the value if null or not.
			output[field].strings.push(s as any);
		}
		return stringLookup.get(s)!;
	}

	let badData = 0;

	pingLoop: for (const ping of data) {
		// First pass checks whether we should store the ping at all. These
		// fields should never be null (based on the query), but we want to
		// check to uphold the invariants of the produced data.
		for (const [k, desc] of pingFields()) {
			if (!desc.nullable && ping[k] === null) {
				console.warn(`Unexpected null in ${k}, omitting ping`);
				badData++;
				continue pingLoop;
			}
		}

		for (const [k, desc] of pingFields()) {
			const inValue = ping[k];

			if (desc.type === PingFieldType.IndexedString) {
				const kfield = k as IndexedStringPingField;
				output[kfield].values.push(getstr(kfield, inValue as string | null));
			} else {
				(output[k] as any[]).push(inValue);
			}
		}
	}

	if (badData > 0) {
		console.warn(`Unexpected data in ${badData} pings`);
	}

	return output;
}