in models/z_buffermodel.py [0:0]
def forward(self, batch):
""" Forward pass of a view synthesis model with a voxel latent field.
"""
# Input values
input_img = batch["images"][0]
output_img = batch["images"][-1]
if "depths" in batch.keys():
depth_img = batch["depths"][0]
# Camera parameters
K = batch["cameras"][0]["K"]
K_inv = batch["cameras"][0]["Kinv"]
input_RT = batch["cameras"][0]["P"]
input_RTinv = batch["cameras"][0]["Pinv"]
output_RT = batch["cameras"][-1]["P"]
output_RTinv = batch["cameras"][-1]["Pinv"]
if torch.cuda.is_available():
input_img = input_img.cuda()
output_img = output_img.cuda()
if "depths" in batch.keys():
depth_img = depth_img.cuda()
K = K.cuda()
K_inv = K_inv.cuda()
input_RT = input_RT.cuda()
input_RTinv = input_RTinv.cuda()
output_RT = output_RT.cuda()
output_RTinv = output_RTinv.cuda()
if self.opt.use_rgb_features:
fs = input_img
else:
fs = self.encoder(input_img)
# Regressed points
if not (self.opt.use_gt_depth):
if not('use_inverse_depth' in self.opt) or not(self.opt.use_inverse_depth):
regressed_pts = (
nn.Sigmoid()(self.pts_regressor(input_img))
* (self.opt.max_z - self.opt.min_z)
+ self.opt.min_z
)
else:
# Use the inverse for datasets with landscapes, where there
# is a long tail on the depth distribution
depth = self.pts_regressor(input_img)
regressed_pts = 1. / (nn.Sigmoid()(depth) * 10 + 0.01)
else:
regressed_pts = depth_img
gen_fs = self.pts_transformer.forward_justpts(
fs,
regressed_pts,
K,
K_inv,
input_RT,
input_RTinv,
output_RT,
output_RTinv,
)
if "modifier" in self.opt.depth_predictor_type:
gen_fs = self.modifier(gen_fs)
gen_img = self.projector(gen_fs)
# And the loss
loss = self.loss_function(gen_img, output_img)
if self.opt.train_depth:
depth_loss = nn.L1Loss()(regressed_pts, depth_img)
loss["Total Loss"] += depth_loss
loss["depth_loss"] = depth_loss
return (
loss,
{
"InputImg": input_img,
"OutputImg": output_img,
"PredImg": gen_img,
"PredDepth": regressed_pts,
},
)