L1 loss is decreasing but classification is increasing in Object Detection

I am trying the Largest object detection and my loss and accuracy are improving in training data. And I have printed out all the L1 Loss, Cross entropy loss, and accuracy for both training and validation data but in Validation L1 loss is decreasing but classification loss is increasing and act is 4-6%. I am using same model resnet34 and exactly the same.

Any suggestions on what I am doing wrong?

model_bb.train()

n_epochs = 20

lr = 0.001
bb_optimizer = torch.optim.Adam(model_bb.parameters(), lr = lr)
val_loss_min = np.Inf
size = 224
total_train_loss = 0
train_total_reg_loss = 0
train_total_cls_loss = 0


total_val_loss = 0
total_val_reg_loss = 0
total_val_cls_loss = 0

for e in tqdm(range(n_epochs), position = 0, leave = True):
    running_loss = 0
    running_reg_loss = 0
    running_cls_loss = 0
    acc = 0
    for img, label in tqdm(train_dl, position = 0, leave = True):
        bb_optimizer.zero_grad()
        img = img.to(device)
        # label = label.to(device)
        label[0] = label[0].to(device)
        label[1] = label[1].to(device)

        pred = model_bb(img)
        loss = combined_loss(pred, label, size)

        running_loss += loss

        running_reg_loss += get_detection_l1_loss(pred[:,:4], label[0], size)

        running_cls_loss += get_detection_cross_entropy_loss(pred[:,4:], label[1])

        acc += get_detection_accuracy(pred[:,4:], label[1])

        loss.backward()

        bb_optimizer.step()

    train_total_loss = running_loss / len(train_dl)

    train_total_reg_loss = running_reg_loss / len(train_dl)

    train_total_cls_loss = running_cls_loss / len(train_dl)

    acc = acc/len(train_dl)

    print("----------TRAINING----------")

    print("Epoch: [{}/{}] \tTotal Loss: {} \tTraining Accuracy: {}".format((e+1), n_epochs, train_total_loss,acc))

                                                                                                             

    model_bb.eval()

    for img, label in tqdm(val_dl, position= 0, leave = True):

        img = img.to(device)
        label[0] = label[0].to(device)
        label[1] = label[1].to(device)

        pred = model_bb(img)
        loss = loss_cls(pred, label)

        r_val += loss

        r_reg_val += get_detection_l1_loss(pred[:,:4], label[0], size)

        r_cls_val += get_detection_cross_entropy_loss(pred[:,4:], label[1])

        r_acc += get_detection_accuracy(pred[:,4:], label[1])

    #--------Validation loss and accuracy![Screenshot (181)|690x387](upload://mYawex1sIu07Ws9QUUkTcTLdfvI.png) 

    total_val_loss = r_val / len(val_dl)

    total_val_reg_loss = r_reg_val / len(val_dl)

    total_val_cls_loss = r_cls_val / len(val_dl)

    acc = r_acc / len(val_dl)

        

    print("----------VALIDATION-----------")

    print("Epoch: [{}/{}]  \tTotal Loss: {} \tAccuracy: {}".format((e+1),n_epochs,total_val_loss,acc))

    

    if total_val_loss < val_loss_min:

        print("Model Saving....")

        torch.save(model_bb.state_dict(),'drive/My Drive/new_files/cls.pt')

        val_loss_min = total_val_loss!

Screenshot (181)|690x387