Merge pull request #1 from goterria/main

Corrections for batch size

Author: sfoucher

.gitignore

@@ -6,11 +6,13 @@ dist/
 build/
 images/
 outputs/
+output/
+DATASETS/
 multirun/
 exp/
 handling/
 tests/
-scripts/
+wandb/
 *.code-workspace
 
 configs/train/*

animaloc/eval/evaluators.py

@@ -200,11 +200,10 @@ def evaluate(self, returns: str = 'recall', wandb_flag: bool = False, viz: bool
                 if i % self.print_freq == 0 or i == len(self.dataloader) - 1:
                     fig = self._vizual(image = images, target = targets, output = output)
                     wandb.log({'validation_vizuals': fig})
-
-            output = self.prepare_feeding(targets, output)
-
-            iter_metrics.feed(**output)
-            iter_metrics.aggregate()
+            for b in range(images.shape[0]):
+                batch_output = self.prepare_feeding(dict(labels= targets['labels'][b], points= targets['points'][b]), (output[0][b].unsqueeze(0), output[1][b].unsqueeze(0)))
+                iter_metrics.feed(**batch_output)
+                iter_metrics.aggregate()
             if log_meters:
                 logger.add_meter('n', sum(iter_metrics.tp) + sum(iter_metrics.fn))
                 logger.add_meter('recall', round(iter_metrics.recall(),2))
@@ -226,8 +225,10 @@ def evaluate(self, returns: str = 'recall', wandb_flag: bool = False, viz: bool
                     })
 
             iter_metrics.flush()
-
-            self.metrics.feed(**output)
+            for b in range(images.shape[0]):
+                batch_output = self.prepare_feeding(dict(labels= targets['labels'][b], points= targets['points'][b]), (output[0][b].unsqueeze(0), output[1][b].unsqueeze(0)))
+                self.metrics.feed(**batch_output)
+            #self.metrics.feed(**output)
 
         self._stored_metrics = self.metrics.copy()
 
@@ -347,14 +348,16 @@ def post_stitcher(self, output: torch.Tensor) -> Any:
 
     def prepare_feeding(self, targets: Dict[str, torch.Tensor], output: List[torch.Tensor]) -> dict:
 
-        gt_coords = [p[::-1] for p in targets['points'].squeeze(0).tolist()]
-        gt_labels = targets['labels'].squeeze(0).tolist()
-        
+        gt_coords = [p[::-1] for p in targets['points'].tolist()]
+        gt_labels = targets['labels'].tolist()
+
+        ndim= numpy.array(gt_coords).ndim
         gt = dict(
             loc = gt_coords,
             labels = gt_labels
         )
 
+
         up = True
         if self.stitcher is not None:
             up = False
@@ -365,8 +368,8 @@ def prepare_feeding(self, targets: Dict[str, torch.Tensor], output: List[torch.T
         preds = dict(
             loc = locs[0],
             labels = labels[0],
-            scores = scores[0],
-            dscores = dscores[0]
+            scores = scores[0], # class scores
+            dscores = dscores[0] # heatmap scores
         )
 
         return dict(gt = gt, preds = preds, est_count = counts[0])

docker/Dockerfile

@@ -1,39 +1,38 @@
-FROM pytorch/pytorch:1.11.0-cuda11.3-cudnn8-runtime AS base
+
+ARG PYTORCH="1.11.0"
+ARG CUDA="11.3"
+ARG CUDNN="8"
+
+FROM pytorch/pytorch:${PYTORCH}-cuda${CUDA}-cudnn${CUDNN}-devel AS base
+ENV TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0+PTX 9.0"
+ENV TORCH_NVCC_FLAGS="-Xfatbin -compress-all"
+
 RUN conda update conda && conda install pip && conda clean -afy
 WORKDIR /herdnet
 COPY environment-dev.yml ./
 RUN conda env update -f environment-dev.yml -n base && conda clean -afy
 
-
-#FROM base as dep_builder
-## Some deps must be built (e.g. against the conda GDAL)
-#RUN apt-get update \
-#    && apt-get install -y gcc build-essential \
-#    && rm -rf /var/lib/apt/lists/*
-#COPY pyproject.toml setup.cfg ./
-#COPY src/stactools/core/__init__.py src/stactools/core/
-## Install dependencies but remove the actual package
-#RUN pip install --prefix=/install .[all] \
-#    && rm -r /install/lib/*/site-packages/stactools*
-
-
 FROM base AS dev
 # Install make for the docs build
-RUN apt-get update \
-    && apt-get install -y gcc make build-essential git \
+# solves a weired problem with NVIDIA with https://github.com/NVIDIA/nvidia-container-toolkit/issues/258#issuecomment-1903945418
+RUN \
+    # Update nvidia GPG key
+    rm /etc/apt/sources.list.d/cuda.list && \
+    rm /etc/apt/sources.list.d/nvidia-ml.list && \
+    apt-key del 7fa2af80 && \
+    apt-get update && apt-get install -y --no-install-recommends wget && \
+    wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb && \
+    dpkg -i cuda-keyring_1.0-1_all.deb && \
+    apt-get update
+
+RUN apt-get update  \
+    && apt-get install -y --no-install-recommends python3-pyqt5 python3-pyqt5.qtwebengine unzip git \
     && rm -rf /var/lib/apt/lists/*
-#COPY --from=dep_builder /install /opt/conda
-#RUN conda install -c conda-forge pandoc && conda clean -af
+
 COPY requirements-dev.txt ./
 RUN pip install -r requirements-dev.txt
 COPY . ./
 # pre-commit run --all-files fails w/o this line
 RUN git init
 RUN pip install -e .
 
-
-#FROM base AS main
-#COPY --from=dep_builder /install /opt/conda
-#COPY src ./src
-#COPY pyproject.toml setup.cfg ./
-#RUN pip install .[all]

docker/build

@@ -17,6 +17,8 @@ Run a console in a docker container with all prerequisites installed.
 
 if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
     docker run --rm -it --gpus all\
+        -v /home/fous3401/DATASETS:/herdnet/DATASETS \
+        --ipc=host \
         -v `pwd`:/herdnet \
         -p 8000:8000 \
         --entrypoint /bin/bash \

docker/run

@@ -11,4 +11,5 @@ pandas
 #opencv-python
 #opencv-python-headless
 hydra-core
-wandb
+wandb
+gdown