threestudio/models/guidance/stable_zero123_guidance.py [141:218]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @torch.cuda.amp.autocast(enabled=False) def set_min_max_steps(self, min_step_percent=0.02, max_step_percent=0.98): self.min_step = int(self.num_train_timesteps * min_step_percent) self.max_step = int(self.num_train_timesteps * max_step_percent) @torch.cuda.amp.autocast(enabled=False) def prepare_embeddings(self, image_path: str) -> None: # load cond image for zero123 assert os.path.exists(image_path) rgba = cv2.cvtColor( cv2.imread(image_path, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGRA2RGBA ) rgba = ( cv2.resize(rgba, (256, 256), interpolation=cv2.INTER_AREA).astype( np.float32 ) / 255.0 ) rgb = rgba[..., :3] * rgba[..., 3:] + (1 - rgba[..., 3:]) self.rgb_256: Float[Tensor, "1 3 H W"] = ( torch.from_numpy(rgb) .unsqueeze(0) .permute(0, 3, 1, 2) .contiguous() .to(self.device) ) self.c_crossattn, self.c_concat = self.get_img_embeds(self.rgb_256) @torch.cuda.amp.autocast(enabled=False) @torch.no_grad() def get_img_embeds( self, img: Float[Tensor, "B 3 256 256"], ) -> Tuple[Float[Tensor, "B 1 768"], Float[Tensor, "B 4 32 32"]]: img = img * 2.0 - 1.0 c_crossattn = self.model.get_learned_conditioning(img.to(self.weights_dtype)) c_concat = self.model.encode_first_stage(img.to(self.weights_dtype)).mode() return c_crossattn, c_concat @torch.cuda.amp.autocast(enabled=False) def encode_images( self, imgs: Float[Tensor, "B 3 256 256"] ) -> Float[Tensor, "B 4 32 32"]: input_dtype = imgs.dtype imgs = imgs * 2.0 - 1.0 latents = self.model.get_first_stage_encoding( self.model.encode_first_stage(imgs.to(self.weights_dtype)) ) return latents.to(input_dtype) # [B, 4, 32, 32] Latent space image @torch.cuda.amp.autocast(enabled=False) def decode_latents( self, latents: Float[Tensor, "B 4 H W"], ) -> Float[Tensor, "B 3 512 512"]: input_dtype = latents.dtype image = self.model.decode_first_stage(latents) image = (image * 0.5 + 0.5).clamp(0, 1) return image.to(input_dtype) @torch.cuda.amp.autocast(enabled=False) @torch.no_grad() def get_cond( self, elevation: Float[Tensor, "B"], azimuth: Float[Tensor, "B"], camera_distances: Float[Tensor, "B"], c_crossattn=None, c_concat=None, **kwargs, ) -> dict: T = torch.stack( [ torch.deg2rad( (90 - elevation) - (90 - self.cfg.cond_elevation_deg) ), # Zero123 polar is 90-elevation torch.sin(torch.deg2rad(azimuth - self.cfg.cond_azimuth_deg)), torch.cos(torch.deg2rad(azimuth - self.cfg.cond_azimuth_deg)), - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - threestudio/models/guidance/zero123_guidance.py [144:221]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @torch.cuda.amp.autocast(enabled=False) def set_min_max_steps(self, min_step_percent=0.02, max_step_percent=0.98): self.min_step = int(self.num_train_timesteps * min_step_percent) self.max_step = int(self.num_train_timesteps * max_step_percent) @torch.cuda.amp.autocast(enabled=False) def prepare_embeddings(self, image_path: str) -> None: # load cond image for zero123 assert os.path.exists(image_path) rgba = cv2.cvtColor( cv2.imread(image_path, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGRA2RGBA ) rgba = ( cv2.resize(rgba, (256, 256), interpolation=cv2.INTER_AREA).astype( np.float32 ) / 255.0 ) rgb = rgba[..., :3] * rgba[..., 3:] + (1 - rgba[..., 3:]) self.rgb_256: Float[Tensor, "1 3 H W"] = ( torch.from_numpy(rgb) .unsqueeze(0) .permute(0, 3, 1, 2) .contiguous() .to(self.device) ) self.c_crossattn, self.c_concat = self.get_img_embeds(self.rgb_256) @torch.cuda.amp.autocast(enabled=False) @torch.no_grad() def get_img_embeds( self, img: Float[Tensor, "B 3 256 256"], ) -> Tuple[Float[Tensor, "B 1 768"], Float[Tensor, "B 4 32 32"]]: img = img * 2.0 - 1.0 c_crossattn = self.model.get_learned_conditioning(img.to(self.weights_dtype)) c_concat = self.model.encode_first_stage(img.to(self.weights_dtype)).mode() return c_crossattn, c_concat @torch.cuda.amp.autocast(enabled=False) def encode_images( self, imgs: Float[Tensor, "B 3 256 256"] ) -> Float[Tensor, "B 4 32 32"]: input_dtype = imgs.dtype imgs = imgs * 2.0 - 1.0 latents = self.model.get_first_stage_encoding( self.model.encode_first_stage(imgs.to(self.weights_dtype)) ) return latents.to(input_dtype) # [B, 4, 32, 32] Latent space image @torch.cuda.amp.autocast(enabled=False) def decode_latents( self, latents: Float[Tensor, "B 4 H W"], ) -> Float[Tensor, "B 3 512 512"]: input_dtype = latents.dtype image = self.model.decode_first_stage(latents) image = (image * 0.5 + 0.5).clamp(0, 1) return image.to(input_dtype) @torch.cuda.amp.autocast(enabled=False) @torch.no_grad() def get_cond( self, elevation: Float[Tensor, "B"], azimuth: Float[Tensor, "B"], camera_distances: Float[Tensor, "B"], c_crossattn=None, c_concat=None, **kwargs, ) -> dict: T = torch.stack( [ torch.deg2rad( (90 - elevation) - (90 - self.cfg.cond_elevation_deg) ), # Zero123 polar is 90-elevation torch.sin(torch.deg2rad(azimuth - self.cfg.cond_azimuth_deg)), torch.cos(torch.deg2rad(azimuth - self.cfg.cond_azimuth_deg)), - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -