in lmms_eval/models/qwen_vl.py [0:0]
def generate_until(self, requests: List[Instance]) -> List[str]:
res = []
def _collate(x):
# the negative sign on len(toks) sorts descending - this has a few advantages:
# - time estimates will always be over not underestimates, which is more useful for planning
# - to know the size of a batch when going through the list, you know the first one is always the batch
# padded context length. this is useful to simplify the batching logic and more importantly to make
# automatic adaptive batches much much easier to implement
# - any OOMs will happen right away rather than near the end
toks = self.tokenizer.encode(x[0])
return -len(toks), x[0]
pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
# we group requests by their generation_kwargs,
# so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling
# in the same batch.
re_ords = utils.Collator([reg.args for reg in requests], _collate, grouping=True)
chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None)
for chunk in chunks:
contexts, all_gen_kwargs, doc_to_visual, doc_id, task, split = zip(*chunk)
task = task[0]
split = split[0]
visuals = [doc_to_visual[0](self.task_dict[task][split][ids]) for ids in doc_id]
visuals = self.flatten(visuals)
visual_paths = []
# save images to /tmp, name generated by hash function
# qwen accept image path. Have to do it here....
for visual in visuals:
name = uuid.uuid4().hex.upper()[0:6]
visual.save(f"/tmp/{name}.png")
visual_paths.append(f"/tmp/{name}.png")
# we assume all gen kwargs in the batch are the same
# this is safe to assume because the `grouper` object ensures it.
gen_kwargs = all_gen_kwargs[0]
# Set default values for until and max_new_tokens
until = [self.tokenizer.decode(self.eot_token_id)]
# Update values from gen_kwargs if present
if "until" in gen_kwargs:
until = gen_kwargs.pop("until")
if isinstance(until, str):
until = [until]
elif not isinstance(until, list):
raise ValueError(f"Expected `gen_kwargs['until']` to be of type Union[str,list] but got {type(until)}")
for i in range(len(contexts)):
if "<image>" in contexts[i]:
context[i] = contexts[i].replace("<image>", "")
questions = [self.prompt.format(visual_path, context) for visual_path, context in zip(visual_paths, contexts)]
# Similar to llava, is visual paths has len 0
# Then nothing will be executed
query = []
for visual_path, context in zip(visual_paths, contexts):
query.append({"image": visual_path})
query.append({"text": context})
if len(visual_paths) == 0:
for context in contexts:
query.append({"text": context})
questions = self.tokenizer.from_list_format(query)
input_ids = self.tokenizer(questions, return_tensors="pt", padding="longest")
# preconfigure gen_kwargs with defaults
if "image_sizes" not in gen_kwargs:
try:
gen_kwargs["image_sizes"] = [visuals[0].size]
except:
gen_kwargs["image_sizes"] = None
if "max_new_tokens" not in gen_kwargs:
gen_kwargs["max_new_tokens"] = 1024
if "temperature" not in gen_kwargs:
gen_kwargs["temperature"] = 0
if "top_p" not in gen_kwargs:
gen_kwargs["top_p"] = None
if "num_beams" not in gen_kwargs:
gen_kwargs["num_beams"] = 1
pad_token_id = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eod_id
cont = self.model.generate(
input_ids.input_ids.to(self.device),
attention_mask=input_ids.attention_mask.to(self.device),
eos_token_id=self.tokenizer.eod_id,
pad_token_id=pad_token_id,
do_sample=True if gen_kwargs["temperature"] > 0 else False,
temperature=gen_kwargs["temperature"],
top_p=gen_kwargs["top_p"],
num_beams=gen_kwargs["num_beams"],
max_new_tokens=gen_kwargs["max_new_tokens"],
use_cache=self.use_cache,
# kwargs=gen_kwargs
)
cont_toks_list = cont.tolist()
for cont_toks, context in zip(cont_toks_list, contexts):
# discard context + left-padding toks if using causal decoder-only LMM
cont_toks = cont_toks[input_ids.input_ids.shape[1] :]
text_outputs = self.tokenizer.decode(cont_toks, skip_special_tokens=True).strip()
for term in until:
if len(term) > 0:
# ignore '' separator,
# for seq2seq case where self.tok_decode(self.eot_token_id) = ''
text_outputs = text_outputs.split(term)[0]
res.append(text_outputs)
self.cache_hook.add_partial("generate_until", (context, gen_kwargs), text_outputs)
# remove visuals from tmp
for visual_path in visual_paths:
try:
os.remove(visual_path)
except:
pass
pbar.update(1)
# reorder this group of results back to original unsorted form
res = re_ords.get_original(res)
pbar.close()
return res