Adding batch validation

.gitignore

@@ -12,9 +12,19 @@ handling/
 tests/
 scripts/
 *.code-workspace
-
+data/
+*.JPG
 configs/train/*
 configs/test/*
 !configs/*/faster_rcnn.yaml
 !configs/*/dla34.yaml
-!configs/*/herdnet.yaml
+!configs/*/herdnet.yaml
+wandb
+runs
+*pt
+nohup.out
+yolo-runs
+tmp
+cache.db
+template_annotations.json
+template_preannotations.json

animaloc/data/batch_utils.py

@@ -36,6 +36,38 @@ def collate_fn(batch):
     batched_images = cat_list(images)
     return batched_images , targets
 
+def val_collate_fn(self, batch: tuple) -> tuple[torch.Tensor, dict]:
+        """collate_fn used to create the validation dataloader
+
+        Args:
+            batch (tuple): (img:torch.Tensor, targets:dict)
+
+        Returns:
+            tuple: (image, target)
+        """
+
+        batched = dict(points=[], labels=[])
+        batch_img = torch.stack([p[0] for p in batch])
+        targets = [p[1] for p in batch]
+        keys = targets[0].keys()
+
+        # get non_empty samples indidces -> set difference
+        non_empty_idx = [i for i, a in enumerate(targets) if len(a["labels"]) > 0]
+        targets_empty = [
+            targets[i] for i in list(set(range(len(batch))) - set(non_empty_idx))
+        ]
+        targets = [targets[i] for i in non_empty_idx]
+
+        # Creating batch
+        for k in keys:
+            batched[k] = []  # initialize to be empty list
+            if k == "points" or k=='labels':
+                batched[k] = [a[k].cpu().tolist() for a in targets]
+                if len(targets_empty) > 0:
+                    batched[k] = batched[k] + [[]] * len(targets_empty)
+        return batch_img, batched
+
+
 def to_xywh(bbox):
     ''' Bbox from [x_min,y_min,x_max,y_max] to [x,y,width,height] '''
     width = bbox[2] - bbox[0]
@@ -83,4 +115,4 @@ def show_batch(sample_batched, denormalize=True):
             ax.add_patch(rect)
 
     ax.set_title(f'Batch of {batch_size} images')
-    ax.axis('off')
+    ax.axis('off')

animaloc/datasets/folder.py

@@ -22,7 +22,6 @@
 
 from typing import Optional, List, Any, Dict
 
-from ..data.types import BoundingBox
 from ..data.utils import group_by_image
 
 from .register import DATASETS
@@ -57,7 +56,8 @@ def __init__(
         csv_file: str, 
         root_dir: str, 
         albu_transforms: Optional[list] = None,
-        end_transforms: Optional[list] = None
+        end_transforms: Optional[list] = None,
+        images_paths:list[str]=None
         ) -> None:
         ''' 
         Args:
@@ -71,12 +71,19 @@ def __init__(
                 tensor and expected target as input and returns a transformed
                 version. These will be applied after albu_transforms. Defaults
                 to None.
+            images_paths (list, optional): list of images to be considered. Overrides root_dir
         '''
 
         super(FolderDataset, self).__init__(csv_file, root_dir, albu_transforms, end_transforms)
 
-        self.folder_images = [i for i in os.listdir(self.root_dir) 
-                                if i.endswith(('.JPG','.jpg','.JPEG','.jpeg'))]
+        if images_paths is None:
+            self.folder_images = [i for i in os.listdir(self.root_dir) 
+                                    if i.endswith(('.JPG','.jpg','.JPEG','.jpeg'))]
+        else:
+            print("Overriding root_dir in FolderDataset, using images_paths")
+            self.folder_images = [i for i in images_paths 
+                                    if i.endswith(('.JPG','.jpg','.JPEG','.jpeg'))]
+
 
         self._img_names = self.folder_images        
         self.anno_keys = self.data.columns

animaloc/eval/evaluators.py

@@ -176,6 +176,17 @@ def evaluate(self, returns: str = 'recall', wandb_flag: bool = False, viz: bool
         Returns:
             float
         '''
+
+        def batch_metrics(metric:Metrics,batchsize:int,output:dict)->None:
+            if batchsize>=1:
+                for i in range(batchsize):
+                    gt = {k:v[i] for k,v in output['gt'].items()}
+                    preds = {k:v[i] for k,v in output['preds'].items()}
+                    counts = output['est_count'][i]
+                    output_i = dict(gt = gt, preds = preds, est_count = counts)
+                    metric.feed(**output_i)
+            else:
+                raise NotImplementedError
 
         self.model.eval()
 
@@ -201,8 +212,12 @@ def evaluate(self, returns: str = 'recall', wandb_flag: bool = False, viz: bool
                     wandb.log({'validation_vizuals': fig})
 
             output = self.prepare_feeding(targets, output)
+
+            batchsize = images.shape[0]
 
-            iter_metrics.feed(**output)
+            batch_metrics(iter_metrics,batchsize,output)
+            # iter_metrics.feed(**output)
+
             iter_metrics.aggregate()
             if log_meters:
                 logger.add_meter('n', sum(iter_metrics.tp) + sum(iter_metrics.fn))
@@ -226,7 +241,8 @@ def evaluate(self, returns: str = 'recall', wandb_flag: bool = False, viz: bool
 
             iter_metrics.flush()
 
-            self.metrics.feed(**output)
+            batch_metrics(self.metrics,batchsize,output)
+            # self.metrics.feed(**output)
 
         self._stored_metrics = self.metrics.copy()
 
@@ -346,8 +362,12 @@ def post_stitcher(self, output: torch.Tensor) -> Any:
 
     def prepare_feeding(self, targets: Dict[str, torch.Tensor], output: List[torch.Tensor]) -> dict:
 
-        gt_coords = [p[::-1] for p in targets['points'].squeeze(0).tolist()]
-        gt_labels = targets['labels'].squeeze(0).tolist()
+        try: # batchsize==1
+            gt_coords = [p[::-1] for p in targets['points'].cpu().tolist()]
+            gt_labels = targets['labels'].cpu().tolist()
+        except Exception: # batchsize>1
+            gt_coords = [p[::-1] for p in targets['points']]
+            gt_labels = targets['labels']
 
         gt = dict(
             loc = gt_coords,
@@ -362,13 +382,14 @@ def prepare_feeding(self, targets: Dict[str, torch.Tensor], output: List[torch.T
         counts, locs, labels, scores, dscores = lmds(output)
 
         preds = dict(
-            loc = locs[0],
-            labels = labels[0],
-            scores = scores[0],
-            dscores = dscores[0]
+            loc = locs,
+            labels = labels,
+            scores = scores,
+            dscores = dscores
         )
 
-        return dict(gt = gt, preds = preds, est_count = counts[0])
+
+        return dict(gt = gt, preds = preds, est_count = counts)
 
 @EVALUATORS.register()
 class DensityMapEvaluator(Evaluator):

animaloc/models/dla.py

@@ -21,7 +21,6 @@
 
 import math
 from os.path import join
-from posixpath import basename
 
 import torch
 from torch import nn

animaloc/models/faster_rcnn.py

@@ -90,7 +90,7 @@ def forward(
         if targets is not None:
             for t in targets:
                 floating_point_types = (torch.float, torch.double, torch.half)
-                if not t["boxes"].dtype in floating_point_types:
+                if t["boxes"].dtype not in floating_point_types:
                     raise TypeError(f"target boxes must of float type, instead got {t['boxes'].dtype}")
                 if not t["labels"].dtype == torch.int64:
                     raise TypeError("target labels must of int64 type, instead got {t['labels'].dtype}")

animaloc/models/herdnet.py

@@ -19,7 +19,6 @@
 
 import torch.nn as nn
 import numpy as np
-import torchvision.transforms as T
 
 from typing import Optional
 

animaloc/train/trainers.py

@@ -22,16 +22,13 @@
 import wandb
 import matplotlib
 
-import matplotlib.pyplot as plt
 matplotlib.use('Agg')
-from torchvision.transforms import ToPILImage
 
 from typing import List, Optional, Union, Callable, Any
 
 from ..utils.torchvision_utils import SmoothedValue, reduce_dict
 from ..utils.logger import CustomLogger
 from ..eval.evaluators import Evaluator
-from ..data.transforms import UnNormalize
 from .adaloss import Adaloss
 
 from ..utils.registry import Registry

animaloc/vizual/objects.py

@@ -17,7 +17,7 @@
 
 import PIL
 
-from PIL import Image, ImageDraw, ImageFont
+from PIL import ImageDraw, ImageFont
 
 __all__ = ['draw_points', 'draw_boxes', 'draw_text']
 

animaloc/vizual/plots.py

@@ -15,14 +15,11 @@
 __version__ = "0.2.0"
 
 
-import torch
 import matplotlib.pyplot as plt 
 import random
 import itertools
 
 from typing import Optional
-from torchvision.transforms import ToPILImage
-from ..data.transforms import UnNormalize, GaussianMap
 
 __all__ = ['PlotPrecisionRecall']
 

my_ml_backend/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.8-slim
+
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y git
+
+ENV PYTHONUNBUFFERED=True \
+    PORT=9090 \
+    WORKERS=2 \
+    THREADS=4
+
+WORKDIR /app
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . ./
+
+CMD exec gunicorn --preload --bind :$PORT --workers $WORKERS --threads $THREADS --timeout 0 _wsgi:app

my_ml_backend/README.md

@@ -0,0 +1,16 @@
+# Quickstart
+
+1. Build and start Machine Learning backend on `http://localhost:9090`
+
+```bash
+docker-compose up
+```
+
+2. Validate that backend is running
+
+```bash
+$ curl http://localhost:9090/health
+{"status":"UP"}
+```
+
+3. Connect to the backend from Label Studio: go to your project `Settings -> Machine Learning -> Add Model` and specify `http://localhost:9090` as a URL.