in backend-apis/app/utils/utils_palm.py [0:0]
def get_embedding(self, text: str = "", image_bytes: bytes = b""):
"""
Args:
text:
image_bytes:
Raises:
ValueError:
Returns:
"""
if not text and not image_bytes:
raise ValueError(
"At least one of text or image_bytes must be specified."
)
instance = struct_pb2.Value()
if text:
instance.struct_value.update({"text": text})
if image_bytes:
encoded_content = base64.b64encode(image_bytes).decode("utf-8")
instance.struct_value.update(
{"image": {"bytesBase64Encoded": encoded_content}}
)
instances = [instance]
endpoint = (
f"projects/{self.project}/locations/{self.location}"
"/publishers/google/models/multimodalembedding@001"
)
response = self.client.predict(endpoint=endpoint, instances=instances)
response_dict = MessageToDict(
response._pb # pylint: disable=protected-access
)
text_embedding = []
if text:
text_emb_value = response_dict["predictions"][0]["textEmbedding"]
text_embedding = [float(v) for v in text_emb_value]
image_embedding = []
if image_bytes:
image_emb_value = response_dict["predictions"][0]["imageEmbedding"]
image_embedding = [float(v) for v in image_emb_value]
return EmbeddingResponse(
text_embedding=text_embedding, image_embedding=image_embedding
)