in lambda/recipes-responder/src/commandline-reindex.ts [105:280]
async function main() {
const staticBucketName = getStaticBucketName();
const contentPrefix = getContentPrefix();
const fastlyApiKey = getFastlyApiKey();
const outgoingEventBus = getOutgoingEventBus();
//Parse the commandline arguments
const {
values: { help, composerId, capiUri, recipeUid, all, test, indexOnly },
} = parseArgs({
options: {
help: {
type: 'boolean',
short: 'h',
},
recipeUid: {
type: 'string',
},
composerId: {
type: 'string',
},
capiUri: {
type: 'string',
},
all: {
type: 'boolean',
short: 'a',
},
test: {
type: 'boolean',
short: 't',
},
indexOnly: {
type: 'boolean',
short: 'i',
},
},
});
if (help) {
console.log(
'Performs a re-index of the specified recipes in the recipe backend. Requires CAPI dev privileges to run.',
);
console.log(
'This expects the following environment variables to be set. You can get the values by running `./get-local-config.sh` and using `source` on the resulting file:',
);
console.log(
" - STACK - deployment stack, required as it's a metrics param",
);
console.log(
' - LAST_UPDATED_INDEX - name of the Dynamo index for querying `lastUpdated',
);
console.log(' - INDEX_TABLE - Dynamo table that holds the index');
console.log(' - STATIC_BUCKET - bucket that holds the static content');
console.log(
' - STAGE - choose whether to target CODE or PROD',
);
console.log(' - CONTENT_URL_BASE - base URL of the Recipes API');
console.log(
' - FASTLY_API_KEY - API key to allow flush of the Fastly cache',
);
console.log(
' - CAPI_KEY - valid Content API key for internal-tier access to the CAPI environment given by the base URL',
);
console.log(
'You must specify exactly one of --recipeUid {uid} / --composerId {composerId} / --capiUri {capi-uri} / --all / --index-only to indicate which content to re-index',
);
process.exit(0);
}
if (!CapiKey || CapiKey == '') {
console.error(
'You need to set the CAPI_KEY environment variable to a valid, internal-tier CAPI key for this to work',
);
process.exit(1);
}
if (process.env['STACK']) {
const msg = `Performing re-index operations on ${
process.env['STAGE'] ?? ''
}`;
if (process.env['STAGE'] == 'PROD') console.error(msg);
else console.log(msg);
console.log('------------------------------------------------------\n');
}
const failedArticleIds: string[] = [];
if (all && !indexOnly) {
const index = await retrieveIndexData();
console.log(
`Re-index all: index was last updated at ${index.lastUpdated.toISOString()}`,
);
const articleIdSet = index.recipes.reduce<Set<string>>(
(idSet, entry) => idSet.add(entry.capiArticleId),
new Set<string>(),
);
const articleIdList = Array.from(articleIdSet.values());
console.log(
`Re-index all: Found ${index.recipes.length} recipes to re-index across ${articleIdList.length} articles`,
);
if (test) {
console.log('Not performing any operations as --test was specified');
} else {
const total = articleIdList.length;
let i = 1;
for (const articleId of articleIdList) {
console.log('------------------------------------------------------');
console.log(`Article ${i} / ${total}...\n`);
const queryUri = await getQueryUri(articleId, undefined, undefined);
try {
await reindex(
queryUri,
staticBucketName,
fastlyApiKey,
contentPrefix,
outgoingEventBus,
);
} catch (e) {
console.error(
`Error reindexing ${queryUri}: ${(e as Error).toString()}`,
);
failedArticleIds.push(queryUri);
}
console.log('------------------------------------------------------\n');
i++;
}
}
} else if (!indexOnly) {
const queryUri = await getQueryUri(capiUri, composerId, recipeUid);
if (test) {
console.log('Not performing any operations as --test was specified');
} else {
await reindex(
queryUri,
staticBucketName,
fastlyApiKey,
contentPrefix,
outgoingEventBus,
);
}
}
console.log('------------------------------------------------------');
console.log('Rebuilding index...');
const indexData = await retrieveIndexData();
console.log('(including all recipes...)');
await writeIndexData({
indexData,
Key: V2_INDEX_JSON,
staticBucketName,
contentPrefix,
fastlyApiKey,
});
console.log('(excluding sponsored recipes...)');
const indexWithoutSponsored: RecipeIndex = {
...indexData,
recipes: indexData.recipes.filter((r) => r.sponsorshipCount === 0),
};
await writeIndexData({
indexData: indexWithoutSponsored,
Key: INDEX_JSON,
staticBucketName,
contentPrefix,
fastlyApiKey,
});
console.log('Finished rebuilding index');
if (failedArticleIds.length > 0) {
console.warn(`${failedArticleIds.length} failed to reindex:`);
failedArticleIds.forEach((capiId) => console.warn(`\t${capiId}`));
}
}