# Copyright 2026 The HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import numpy as np import torch import torch.nn as nn from ..utils import is_accelerate_available, is_scipy_available, is_vision_available from .loss_for_object_detection import ( HungarianMatcher, _set_aux_loss, box_iou, dice_loss, generalized_box_iou, nested_tensor_from_tensor_list, sigmoid_focal_loss, ) if is_vision_available(): from transformers.image_transforms import center_to_corners_format if is_scipy_available(): from scipy.optimize import linear_sum_assignment if is_accelerate_available(): from accelerate import PartialState from accelerate.utils import reduce class LwDetrHungarianMatcher(HungarianMatcher): @torch.no_grad() def forward(self, outputs, targets, group_detr): """ Differences: - out_prob = outputs["logits"].flatten(0, 1).sigmoid() instead of softmax - class_cost uses alpha and gamma """ batch_size, num_queries = outputs["logits"].shape[:2] # We flatten to compute the cost matrices in a batch out_prob = outputs["logits"].flatten(0, 1).sigmoid() # [batch_size * num_queries, num_classes] out_bbox = outputs["pred_boxes"].flatten(0, 1) # [batch_size * num_queries, 4] # Also concat the target labels and boxes target_ids = torch.cat([v["class_labels"] for v in targets]) target_bbox = torch.cat([v["boxes"] for v in targets]) # Compute the classification cost. alpha = 0.25 gamma = 2.0 neg_cost_class = (1 - alpha) * (out_prob**gamma) * (-(1 - out_prob + 1e-8).log()) pos_cost_class = alpha * ((1 - out_prob) ** gamma) * (-(out_prob + 1e-8).log()) class_cost = pos_cost_class[:, target_ids] - neg_cost_class[:, target_ids] # Compute the L1 cost between boxes, cdist only supports float32 dtype = out_bbox.dtype out_bbox = out_bbox.to(torch.float32) target_bbox = target_bbox.to(torch.float32) bbox_cost = torch.cdist(out_bbox, target_bbox, p=1) bbox_cost = bbox_cost.to(dtype) # Compute the giou cost between boxes giou_cost = -generalized_box_iou(center_to_corners_format(out_bbox), center_to_corners_format(target_bbox)) # Final cost matrix cost_matrix = self.bbox_cost * bbox_cost + self.class_cost * class_cost + self.giou_cost * giou_cost cost_matrix = cost_matrix.view(batch_size, num_queries, -1).cpu() sizes = [len(v["boxes"]) for v in targets] indices = [] group_num_queries = num_queries // group_detr cost_matrix_list = cost_matrix.split(group_num_queries, dim=1) for group_id in range(group_detr): group_cost_matrix = cost_matrix_list[group_id] group_indices = [linear_sum_assignment(c[i]) for i, c in enumerate(group_cost_matrix.split(sizes, -1))] if group_id == 0: indices = group_indices else: indices = [ ( np.concatenate([indice1[0], indice2[0] + group_num_queries * group_id]), np.concatenate([indice1[1], indice2[1]]), ) for indice1, indice2 in zip(indices, group_indices) ] return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices] class LwDetrImageLoss(nn.Module): def __init__(self, matcher, num_classes, focal_alpha, losses, group_detr): super().__init__() self.matcher = matcher self.num_classes = num_classes self.focal_alpha = focal_alpha self.losses = losses self.group_detr = group_detr # removed logging parameter, which was part of the original implementation def loss_labels(self, outputs, targets, indices, num_boxes): if "logits" not in outputs: raise KeyError("No logits were found in the outputs") source_logits = outputs["logits"] idx = self._get_source_permutation_idx(indices) target_classes_o = torch.cat([t["class_labels"][J] for t, (_, J) in zip(targets, indices)]) alpha = self.focal_alpha gamma = 2 src_boxes = outputs["pred_boxes"][idx] target_boxes = torch.cat([t["boxes"][i] for t, (_, i) in zip(targets, indices)], dim=0) iou_targets = torch.diag( box_iou(center_to_corners_format(src_boxes.detach()), center_to_corners_format(target_boxes))[0] ) # Convert to the same dtype as the source logits as box_iou upcasts to float32 iou_targets = iou_targets.to(source_logits.dtype) pos_ious = iou_targets.clone().detach() prob = source_logits.sigmoid() # init positive weights and negative weights pos_weights = torch.zeros_like(source_logits) neg_weights = prob**gamma pos_ind = list(idx) pos_ind.append(target_classes_o) t = prob[pos_ind].pow(alpha) * pos_ious.pow(1 - alpha) t = torch.clamp(t, 0.01).detach() pos_weights[pos_ind] = t neg_weights[pos_ind] = 1 - t loss_ce = -pos_weights * prob.log() - neg_weights * (1 - prob).log() loss_ce = loss_ce.sum() / num_boxes losses = {"loss_ce": loss_ce} return losses # Copied from loss.loss_for_object_detection.ImageLoss.loss_cardinality @torch.no_grad() def loss_cardinality(self, outputs, targets, indices, num_boxes): """ Compute the cardinality error, i.e. the absolute error in the number of predicted non-empty boxes. This is not really a loss, it is intended for logging purposes only. It doesn't propagate gradients. """ logits = outputs["logits"] device = logits.device target_lengths = torch.as_tensor([len(v["class_labels"]) for v in targets], device=device) # Count the number of predictions that are NOT "no-object" (which is the last class) card_pred = (logits.argmax(-1) != logits.shape[-1] - 1).sum(1) card_err = nn.functional.l1_loss(card_pred.float(), target_lengths.float()) losses = {"cardinality_error": card_err} return losses # Copied from loss.loss_for_object_detection.ImageLoss.loss_boxes def loss_boxes(self, outputs, targets, indices, num_boxes): """ Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss. Targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4]. The target boxes are expected in format (center_x, center_y, w, h), normalized by the image size. """ if "pred_boxes" not in outputs: raise KeyError("No predicted boxes found in outputs") idx = self._get_source_permutation_idx(indices) source_boxes = outputs["pred_boxes"][idx] target_boxes = torch.cat([t["boxes"][i] for t, (_, i) in zip(targets, indices)], dim=0) loss_bbox = nn.functional.l1_loss(source_boxes, target_boxes, reduction="none") losses = {} losses["loss_bbox"] = loss_bbox.sum() / num_boxes loss_giou = 1 - torch.diag( generalized_box_iou(center_to_corners_format(source_boxes), center_to_corners_format(target_boxes)) ) losses["loss_giou"] = loss_giou.sum() / num_boxes return losses # Copied from loss.loss_for_object_detection.ImageLoss.loss_masks def loss_masks(self, outputs, targets, indices, num_boxes): """ Compute the losses related to the masks: the focal loss and the dice loss. Targets dicts must contain the key "masks" containing a tensor of dim [nb_target_boxes, h, w]. """ if "pred_masks" not in outputs: raise KeyError("No predicted masks found in outputs") source_idx = self._get_source_permutation_idx(indices) target_idx = self._get_target_permutation_idx(indices) source_masks = outputs["pred_masks"] source_masks = source_masks[source_idx] masks = [t["masks"] for t in targets] # TODO use valid to mask invalid areas due to padding in loss target_masks, valid = nested_tensor_from_tensor_list(masks).decompose() target_masks = target_masks.to(source_masks) target_masks = target_masks[target_idx] # upsample predictions to the target size source_masks = nn.functional.interpolate( source_masks[:, None], size=target_masks.shape[-2:], mode="bilinear", align_corners=False ) source_masks = source_masks[:, 0].flatten(1) target_masks = target_masks.flatten(1) target_masks = target_masks.view(source_masks.shape) losses = { "loss_mask": sigmoid_focal_loss(source_masks, target_masks, num_boxes), "loss_dice": dice_loss(source_masks, target_masks, num_boxes), } return losses # Copied from loss.loss_for_object_detection.ImageLoss._get_source_permutation_idx def _get_source_permutation_idx(self, indices): # permute predictions following indices batch_idx = torch.cat([torch.full_like(source, i) for i, (source, _) in enumerate(indices)]) source_idx = torch.cat([source for (source, _) in indices]) return batch_idx, source_idx # Copied from loss.loss_for_object_detection.ImageLoss._get_target_permutation_idx def _get_target_permutation_idx(self, indices): # permute targets following indices batch_idx = torch.cat([torch.full_like(target, i) for i, (_, target) in enumerate(indices)]) target_idx = torch.cat([target for (_, target) in indices]) return batch_idx, target_idx def get_loss(self, loss, outputs, targets, indices, num_boxes): loss_map = { "labels": self.loss_labels, "cardinality": self.loss_cardinality, "boxes": self.loss_boxes, "masks": self.loss_masks, } if loss not in loss_map: raise ValueError(f"Loss {loss} not supported") return loss_map[loss](outputs, targets, indices, num_boxes) def forward(self, outputs, targets): """ This performs the loss computation. Args: outputs (`dict`, *optional*): Dictionary of tensors, see the output specification of the model for the format. targets (`list[dict]`, *optional*): List of dicts, such that `len(targets) == batch_size`. The expected keys in each dict depends on the losses applied, see each loss' doc. """ group_detr = self.group_detr if self.training else 1 outputs_without_aux_and_enc = { k: v for k, v in outputs.items() if k != "enc_outputs" and k != "auxiliary_outputs" } # Retrieve the matching between the outputs of the last layer and the targets indices = self.matcher(outputs_without_aux_and_enc, targets, group_detr) # Compute the average number of target boxes across all nodes, for normalization purposes num_boxes = sum(len(t["class_labels"]) for t in targets) num_boxes = num_boxes * group_detr num_boxes = torch.as_tensor([num_boxes], dtype=torch.float, device=next(iter(outputs.values())).device) world_size = 1 if is_accelerate_available(): if PartialState._shared_state != {}: num_boxes = reduce(num_boxes) world_size = PartialState().num_processes num_boxes = torch.clamp(num_boxes / world_size, min=1).item() # Compute all the requested losses losses = {} for loss in self.losses: losses.update(self.get_loss(loss, outputs, targets, indices, num_boxes)) # In case of auxiliary losses, we repeat this process with the output of each intermediate layer. if "auxiliary_outputs" in outputs: for i, auxiliary_outputs in enumerate(outputs["auxiliary_outputs"]): indices = self.matcher(auxiliary_outputs, targets, group_detr) for loss in self.losses: if loss == "masks": # Intermediate masks losses are too costly to compute, we ignore them. continue l_dict = self.get_loss(loss, auxiliary_outputs, targets, indices, num_boxes) l_dict = {k + f"_{i}": v for k, v in l_dict.items()} losses.update(l_dict) if "enc_outputs" in outputs: enc_outputs = outputs["enc_outputs"] indices = self.matcher(enc_outputs, targets, group_detr=group_detr) for loss in self.losses: l_dict = self.get_loss(loss, enc_outputs, targets, indices, num_boxes) l_dict = {k + "_enc": v for k, v in l_dict.items()} losses.update(l_dict) return losses def LwDetrForObjectDetectionLoss( logits, labels, device, pred_boxes, config, outputs_class=None, outputs_coord=None, enc_outputs_class=None, enc_outputs_coord=None, **kwargs, ): # First: create the matcher matcher = LwDetrHungarianMatcher( class_cost=config.class_cost, bbox_cost=config.bbox_cost, giou_cost=config.giou_cost ) # Second: create the criterion losses = ["labels", "boxes", "cardinality"] criterion = LwDetrImageLoss( matcher=matcher, num_classes=config.num_labels, focal_alpha=config.focal_alpha, losses=losses, group_detr=config.group_detr, ) criterion.to(device) # Third: compute the losses, based on outputs and labels outputs_loss = {} auxiliary_outputs = None outputs_loss["logits"] = logits outputs_loss["pred_boxes"] = pred_boxes outputs_loss["enc_outputs"] = { "logits": enc_outputs_class, "pred_boxes": enc_outputs_coord, } if config.auxiliary_loss: auxiliary_outputs = _set_aux_loss(outputs_class, outputs_coord) outputs_loss["auxiliary_outputs"] = auxiliary_outputs loss_dict = criterion(outputs_loss, labels) # Fourth: compute total loss, as a weighted sum of the various losses weight_dict = {"loss_ce": 1, "loss_bbox": config.bbox_loss_coefficient} weight_dict["loss_giou"] = config.giou_loss_coefficient if config.auxiliary_loss: aux_weight_dict = {} for i in range(config.decoder_layers - 1): aux_weight_dict.update({k + f"_{i}": v for k, v in weight_dict.items()}) weight_dict.update(aux_weight_dict) enc_weight_dict = {k + "_enc": v for k, v in weight_dict.items()} weight_dict.update(enc_weight_dict) loss = sum(loss_dict[k] * weight_dict[k] for k in loss_dict if k in weight_dict) return loss, loss_dict, auxiliary_outputs