downloader/convert_dataset.py : Added better typing and formatting

.gitignore

@@ -1,6 +1,6 @@
 data
 */models*
-venv
+.venv
 */out*
 python/__pycache__
 cpp/bin

cpp/Dockerfile

@@ -1,11 +1,11 @@
-FROM nvidia/cuda:12.5.1-devel-ubi9 AS builder
+FROM nvidia/cuda:12.6.2-devel-ubi9 AS builder
 
 WORKDIR /home/ViolaJones/cpp
 
 COPY *.cu *.cpp *.hpp Makefile ./
 RUN make -j "$(nproc)" && make -j "$(nproc)" ./bin/ViolaJonesTest
 
-FROM nvidia/cuda:12.5.1-base-ubi9
+FROM nvidia/cuda:12.6.2-base-ubi9
 
 WORKDIR /home/ViolaJones/cpp
 

cpp/docker-compose.yaml

@@ -12,5 +12,4 @@ services:
         reservations:
           devices:
             - driver: nvidia
-              count: 1
               capabilities: [gpu]

downloader/Dockerfile

@@ -1,6 +1,6 @@
-FROM alpine:3.20.1
+FROM alpine:3.20.3
 
-RUN apk add --no-cache curl=8.8.0-r0 python3=3.12.3-r1 && rm -rf /var/cache/apk*
+RUN apk add --no-cache curl=8.11.0-r1 python3=3.12.7-r0 && rm -rf /var/cache/apk*
 
 WORKDIR /home/ViolaJones/downloader
 COPY requirements.txt activate.sh ./

downloader/activate.sh

@@ -4,16 +4,14 @@
 set -e
 
 test -z "$EXEC_DIR" && EXEC_DIR=.
-test -z "$VENV_PATH" && VENV_PATH="$EXEC_DIR/venv"
+test -z "$VENV_PATH" && VENV_PATH="$EXEC_DIR/.venv"
 
 activate(){
 	if [ ! -d "$VENV_PATH" ]; then
 		echo 'Creating python virtual environnement'
-		python -m venv "$VENV_PATH"
+		python -m venv --upgrade-deps "$VENV_PATH"
 		echo 'Activating virtual environnement'
 		activate
-		echo 'Updating base pip packages'
-		python -m pip install -U setuptools pip
 		echo 'Installing requirements'
 		pip install -r requirements.txt
 	elif [ -f "$VENV_PATH"/Scripts/activate ]; then . "$VENV_PATH"/Scripts/activate

downloader/convert_dataset.py

@@ -3,29 +3,31 @@ from tqdm import tqdm
 from functools import partial
 from sys import argv
 import numpy as np
+from numpy.typing import NDArray
+from typing import Final, Callable
 from os import path, listdir
 
 # Induce determinism
-np.random.seed(133742)
+np.random.seed(196_863)
 
-# Makes the "leave" argument default to False
-tqdm = partial(tqdm, leave = False)
+# Makes the 'leave' argument default to False
+tqdm: Callable = partial(tqdm, leave = False)
 
-def read_pgm(pgm_file: BufferedReader) -> np.ndarray:
+def read_pgm(pgm_file: BufferedReader) -> NDArray[np.uint8]:
 	"""Read the data of a PGM file
 
 	Args:
 		pgm_file (BufferedReader): PGM File
 
 	Returns:
-		np.ndarray: PGM data
+		NDArray[np.uint8]: PGM data
 	"""
-	assert (f := pgm_file.readline()) == b'P5\n', f"Incorrect file format: {f}"
-	(width, height) = [int(i) for i in pgm_file.readline().split()]
-	assert width > 0 and height > 0, f"Incorrect dimensions: {width}x{height}"
-	assert (depth := int(pgm_file.readline())) < 256, f"Incorrect depth: {depth}"
+	assert (f := pgm_file.readline()) == b'P5\n', f'Incorrect file format: {f}'
+	(width, height) = (int(i) for i in pgm_file.readline().split())
+	assert width > 0 and height > 0, f'Incorrect dimensions: {width}x{height}'
+	assert (depth := int(pgm_file.readline())) < 256, f'Incorrect depth: {depth}'
 
-	buff = np.empty(height * width, dtype = np.uint8)
+	buff: Final[NDArray[np.uint8]] = np.empty(height * width, dtype = np.uint8)
 	for i in range(buff.shape[0]):
 		buff[i] = ord(pgm_file.read(1))
 	return buff.reshape((height, width))
@@ -36,25 +38,25 @@ def __main__(data_path: str) -> None:
 	Args:
 		data_path (str): Path of the PGM files
 	"""
-	for set_name in tqdm(["train", "test"], desc = "set name"):
+	for set_name in tqdm(['train', 'test'], desc = 'set name'):
 		X, y = [], []
-		for y_i, label in enumerate(tqdm(["non-face", "face"], desc = "label")):
-			for filename in tqdm(listdir(f"{data_path}/{set_name}/{label}"), desc = "Reading pgm file"):
-				with open(f"{data_path}/{set_name}/{label}/{filename}", "rb") as face:
+		for y_i, label in enumerate(tqdm(['non-face', 'face'], desc = 'label')):
+			for filename in tqdm(listdir(f'{data_path}/{set_name}/{label}'), desc = 'Reading pgm file'):
+				with open(f'{data_path}/{set_name}/{label}/{filename}', 'rb') as face:
 					X.append(read_pgm(face))
 					y.append(y_i)
 
 		X, y = np.asarray(X), np.asarray(y)
-		idx = np.random.permutation(y.shape[0])
+		idx: NDArray[np.int64] = np.random.permutation(y.shape[0])
 		X, y = X[idx], y[idx]
 
-		for org, s in tqdm(zip("Xy", [X, y]), desc = f"Writing {set_name}"):
-			with open(f"{data_path}/{org}_{set_name}.bin", "w") as out:
-				out.write(f'{str(s.shape)[1:-1].replace(",", "")}\n')
-				raw = s.ravel()
-				for s_i in tqdm(raw[:-1], desc = f"Writing {org}"):
-					out.write(f"{s_i} ")
+		for org, s in tqdm(zip('Xy', [X, y]), desc = f'Writing {set_name}'):
+			with open(f'{data_path}/{org}_{set_name}.bin', 'w') as out:
+				out.write(f'{str(s.shape)[1:-1].replace(',', '')}\n')
+				raw: NDArray = s.ravel()
+				for s_i in tqdm(raw[:-1], desc = f'Writing {org}'):
+					out.write(f'{s_i} ')
 				out.write(str(raw[-1]))
 
-if __name__ == "__main__":
-	__main__(argv[1]) if len(argv) == 2 else print(f"Usage: python {__file__[__file__.rfind(path.sep) + 1:]} ./data_location")
+if __name__ == '__main__':
+	__main__(argv[1]) if len(argv) == 2 else print(f'Usage: python {__file__[__file__.rfind(path.sep) + 1:]} ./data_location')

downloader/requirements.txt

@@ -1,2 +1,2 @@
-numpy==2.0.0
-tqdm==4.66.4
+numpy==2.1.3
+tqdm==4.67.0

python/Dockerfile

@@ -1,6 +1,6 @@
-FROM nvidia/cuda:12.5.1-devel-ubi9 AS builder
+FROM nvidia/cuda:12.6.2-devel-ubi9
 
-RUN dnf install -y python3.12-3.12.1-4.el9 \
+RUN dnf install -y python3.12-3.12.1-4.el9_4.4 \
 	&& dnf clean all \
 	&& ln -s /usr/bin/python3.12 /usr/bin/python
 

python/Makefile

@@ -57,7 +57,7 @@ clean:
 
 .PHONY: mrproper
 mrproper: clean
-	@rm -rfv __pycache__ venv
+	@rm -rfv __pycache__ .venv
 
 .PHONY: help
 help:

python/activate.sh

@@ -4,16 +4,14 @@
 set -e
 
 test -z "$EXEC_DIR" && EXEC_DIR=.
-test -z "$VENV_PATH" && VENV_PATH="$EXEC_DIR/venv"
+test -z "$VENV_PATH" && VENV_PATH="$EXEC_DIR/.venv"
 
 activate(){
 	if [ ! -d "$VENV_PATH" ]; then
 		echo 'Creating python virtual environnement'
-		python -m venv "$VENV_PATH"
+		python -m venv --upgrade-deps "$VENV_PATH"
 		echo 'Activating virtual environnement'
 		activate
-		echo 'Updating base pip packages'
-		python -m pip install -U setuptools pip
 		echo 'Installing requirements'
 		pip install -r requirements.txt
 	elif [ -f "$VENV_PATH"/Scripts/activate ]; then . "$VENV_PATH"/Scripts/activate

python/docker-compose.yaml

@@ -12,5 +12,4 @@ services:
         reservations:
           devices:
             - driver: nvidia
-              count: 1
               capabilities: [gpu]

python/requirements.txt

@@ -1,3 +1,5 @@
 numba==0.60.0
-scikit-learn==1.5.1
-tqdm==4.66.4
+scikit-learn==1.5.2
+tqdm==4.67.0
+
+#pudb==2024.1.3