export async function endpointBedrock()

in src/lib/server/endpoints/aws/endpointBedrock.ts [35:136]
62 lines of code
5 McCabe index (conditional complexity)

export async function endpointBedrock(
	input: z.input<typeof endpointBedrockParametersSchema>
): Promise<Endpoint> {
	const { region, model, anthropicVersion, multimodal, isNova } =
		endpointBedrockParametersSchema.parse(input);

	let BedrockRuntimeClient, InvokeModelWithResponseStreamCommand;
	try {
		({ BedrockRuntimeClient, InvokeModelWithResponseStreamCommand } = await import(
			"@aws-sdk/client-bedrock-runtime"
		));
	} catch (error) {
		throw new Error("Failed to import @aws-sdk/client-bedrock-runtime. Make sure it's installed.");
	}

	const client = new BedrockRuntimeClient({
		region,
	});
	const imageProcessor = makeImageProcessor(multimodal.image);

	return async ({ messages, preprompt, generateSettings }) => {
		let system = preprompt;
		// Use the first message as the system prompt if it's of type "system"
		if (messages?.[0]?.from === "system") {
			system = messages[0].content;
			messages = messages.slice(1); // Remove the first system message from the array
		}

		const formattedMessages = await prepareMessages(messages, isNova, imageProcessor);

		let tokenId = 0;
		const parameters = { ...model.parameters, ...generateSettings };
		return (async function* () {
			const baseCommandParams = {
				contentType: "application/json",
				accept: "application/json",
				modelId: model.id,
			};

			const maxTokens = parameters.max_new_tokens || 4096;

			let bodyContent;
			if (isNova) {
				bodyContent = {
					messages: formattedMessages,
					inferenceConfig: {
						maxTokens,
						topP: 0.1,
						temperature: 1.0,
					},
					system: [{ text: system }],
				};
			} else {
				bodyContent = {
					anthropic_version: anthropicVersion,
					max_tokens: maxTokens,
					messages: formattedMessages,
					system,
				};
			}

			const command = new InvokeModelWithResponseStreamCommand({
				...baseCommandParams,
				body: Buffer.from(JSON.stringify(bodyContent), "utf-8"),
				trace: "DISABLED",
			});

			const response = await client.send(command);

			let text = "";

			for await (const item of response.body ?? []) {
				const chunk = JSON.parse(new TextDecoder().decode(item.chunk?.bytes));
				if ("contentBlockDelta" in chunk || chunk.type === "content_block_delta") {
					const chunkText = chunk.contentBlockDelta?.delta?.text || chunk.delta?.text || "";
					text += chunkText;
					yield {
						token: {
							id: tokenId++,
							text: chunkText,
							logprob: 0,
							special: false,
						},
						generated_text: null,
						details: null,
					} satisfies TextGenerationStreamOutput;
				} else if ("messageStop" in chunk || chunk.type === "message_stop") {
					yield {
						token: {
							id: tokenId++,
							text: "",
							logprob: 0,
							special: true,
						},
						generated_text: text,
						details: null,
					} satisfies TextGenerationStreamOutput;
				}
			}
		})();
	};
}