saundersp/ViolaJones
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

python : Updated code with better display, documentation and format_time

Browse Source
This commit is contained in:
saundersp 2024-04-28 00:25:13 +02:00
parent c7d21e1014
commit 718724b63b
11 changed files with 591 additions and 566 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
python/ViolaJones.py
View File

@ -21,10 +21,10 @@ def build_features(width: int, height: int) -> np.ndarray:
"""Initialize the features base on the input shape. """Initialize the features base on the input shape.
Args: Args:
shape (Tuple[int, int]): Shape of the image (Width, Height). shape (Tuple[int, int]): Shape of the image (Width, Height)
Returns: Returns:
np.ndarray: The initialized features. np.ndarray: The initialized features
""" """
feats = [] feats = []
empty = (0, 0, 0, 0) empty = (0, 0, 0, 0)
@ -63,10 +63,10 @@ def init_weights(y_train: np.ndarray) -> np.ndarray:
"""Initialize the weights of the weak classifiers based on the training labels. """Initialize the weights of the weak classifiers based on the training labels.
Args: Args:
y_train (np.ndarray): Training labels. y_train (np.ndarray): Training labels
Returns: Returns:
np.ndarray: The initialized weights. np.ndarray: The initialized weights
""" """
weights = np.empty_like(y_train, dtype = np.float64) weights = np.empty_like(y_train, dtype = np.float64)
t = y_train.sum() t = y_train.sum()
@ -79,12 +79,12 @@ def classify_weak_clf(x_feat_i: np.ndarray, threshold: int, polarity: int) -> np
"""Classify the integrated features based on polarity and threshold. """Classify the integrated features based on polarity and threshold.
Args: Args:
x_feat_i (np.ndarray): Integrated features. x_feat_i (np.ndarray): Integrated features
threshold (int): Trained threshold. threshold (int): Trained threshold
polarity (int): Trained polarity. polarity (int): Trained polarity
Returns: Returns:
np.ndarray: Classified features. np.ndarray: Classified features
""" """
res = np.zeros_like(x_feat_i, dtype = np.int8) res = np.zeros_like(x_feat_i, dtype = np.int8)
res[polarity * x_feat_i < polarity * threshold] = 1 res[polarity * x_feat_i < polarity * threshold] = 1
@ -95,10 +95,10 @@ def select_best(classifiers: np.ndarray, weights: np.ndarray, X_feat: np.ndarray
"""Select the best classifier given theirs predictions. """Select the best classifier given theirs predictions.
Args: Args:
classifiers (np.ndarray): The weak classifiers. classifiers (np.ndarray): The weak classifiers
weights (np.ndarray): Trained weights of each classifiers. weights (np.ndarray): Trained weights of each classifiers
X_feat (np.ndarray): Integrated features. X_feat (np.ndarray): Integrated features
y (np.ndarray): Features labels. y (np.ndarray): Features labels
Returns: Returns:
Tuple[int, float, np.ndarray]: Index of the best classifier, the best error and the best accuracy Tuple[int, float, np.ndarray]: Index of the best classifier, the best error and the best accuracy
@ -116,13 +116,13 @@ def train_viola_jones(T: int, X_feat: np.ndarray, X_feat_argsort: np.ndarray, y:
"""Train the weak classifiers. """Train the weak classifiers.
Args: Args:
T (int): Number of weak classifiers. T (int): Number of weak classifiers
X_feat (np.ndarray): Integrated features. X_feat (np.ndarray): Integrated features
X_feat_argsort (np.ndarray): Sorted indexes of the integrated features. X_feat_argsort (np.ndarray): Sorted indexes of the integrated features
y (np.ndarray): Features labels. y (np.ndarray): Features labels
Returns: Returns:
Tuple[np.ndarray, np.ndarray]: List of trained alphas and the list of the final classifiers. Tuple[np.ndarray, np.ndarray]: List of trained alphas and the list of the final classifiers
""" """
weights = init_weights(y) weights = init_weights(y)
alphas, final_classifier = np.empty(T, dtype = np.float64), np.empty((T, 3), dtype = np.int32) alphas, final_classifier = np.empty(T, dtype = np.float64), np.empty((T, 3), dtype = np.int32)
@ -144,12 +144,12 @@ def classify_viola_jones(alphas: np.ndarray, classifiers: np.ndarray, X_feat: np
"""Classify the trained classifiers on the given features. """Classify the trained classifiers on the given features.
Args: Args:
alphas (np.ndarray): Trained alphas. alphas (np.ndarray): Trained alphas
classifiers (np.ndarray): Trained classifiers. classifiers (np.ndarray): Trained classifiers
X_feat (np.ndarray): Integrated features. X_feat (np.ndarray): Integrated features
Returns: Returns:
np.ndarray: Classification results. np.ndarray: Classification results
""" """
total = np.zeros(X_feat.shape[1], dtype = np.float64) total = np.zeros(X_feat.shape[1], dtype = np.float64)
@ -161,22 +161,22 @@ def classify_viola_jones(alphas: np.ndarray, classifiers: np.ndarray, X_feat: np
y_pred[total >= 0.5 * np.sum(alphas)] = 1 y_pred[total >= 0.5 * np.sum(alphas)] = 1
return y_pred return y_pred
@njit #@njit
def get_best_anova_features(X: np.ndarray, y: np.ndarray) -> np.ndarray: #def get_best_anova_features(X: np.ndarray, y: np.ndarray) -> np.ndarray:
#SelectPercentile(f_classif, percentile = 10).fit(X, y).get_support(indices = True) # #SelectPercentile(f_classif, percentile = 10).fit(X, y).get_support(indices = True)
classes = [X.T[y == 0].astype(np.float64), X.T[y == 1].astype(np.float64)] # classes = [X.T[y == 0].astype(np.float64), X.T[y == 1].astype(np.float64)]
n_samples_per_class = np.asarray([classes[0].shape[0], classes[1].shape[0]]) # n_samples_per_class = np.asarray([classes[0].shape[0], classes[1].shape[0]])
n_samples = classes[0].shape[0] + classes[1].shape[0] # n_samples = classes[0].shape[0] + classes[1].shape[0]
ss_alldata = (classes[0] ** 2).sum(axis = 0) + (classes[1] ** 2).sum(axis = 0) # ss_all_data = (classes[0] ** 2).sum(axis = 0) + (classes[1] ** 2).sum(axis = 0)
sums_classes = [np.asarray(classes[0].sum(axis = 0)), np.asarray(classes[1].sum(axis = 0))] # sums_classes = [np.asarray(classes[0].sum(axis = 0)), np.asarray(classes[1].sum(axis = 0))]
sq_of_sums_alldata = (sums_classes[0] + sums_classes[1]) ** 2 # sq_of_sums_all_data = (sums_classes[0] + sums_classes[1]) ** 2
sq_of_sums_args = [sums_classes[0] ** 2, sums_classes[1] ** 2] # sq_of_sums_args = [sums_classes[0] ** 2, sums_classes[1] ** 2]
ss_tot = ss_alldata - sq_of_sums_alldata / n_samples # ss_tot = ss_all_data - sq_of_sums_all_data / n_samples
#
sqd_sum_bw_n = sq_of_sums_args[0] / n_samples_per_class[0] + \ # sqd_sum_bw_n = sq_of_sums_args[0] / n_samples_per_class[0] + \
sq_of_sums_args[1] / n_samples_per_class[1] - sq_of_sums_alldata / n_samples # sq_of_sums_args[1] / n_samples_per_class[1] - sq_of_sums_all_data / n_samples
ss_wn = ss_tot - sqd_sum_bw_n # ss_wn = ss_tot - sqd_sum_bw_n
df_wn = n_samples - 2 # df_wn = n_samples - 2
msw = ss_wn / df_wn # msw = ss_wn / df_wn
f_values = sqd_sum_bw_n / msw # f_values = sqd_sum_bw_n / msw
return np.sort(np.argsort(f_values)[::-1][: int(np.ceil(X.shape[0] / 10.0))]) # return np.sort(np.argsort(f_values)[::-1][: int(np.ceil(X.shape[0] / 10.0))])

161
python/ViolaJonesCPU.py
View File

@ -18,10 +18,10 @@ def set_integral_image(X: np.ndarray) -> np.ndarray:
"""Transform the input images in integrated images (CPU version). """Transform the input images in integrated images (CPU version).
Args: Args:
X (np.ndarray): Dataset of images. X (np.ndarray): Dataset of images
Returns: Returns:
np.ndarray: Dataset of integrated images. np.ndarray: Dataset of integrated images
""" """
X_ii = np.empty_like(X, dtype = np.uint32) X_ii = np.empty_like(X, dtype = np.uint32)
for i, Xi in enumerate(tqdm_iter(X, "Applying integral image")): for i, Xi in enumerate(tqdm_iter(X, "Applying integral image")):
@ -34,59 +34,18 @@ def set_integral_image(X: np.ndarray) -> np.ndarray:
X_ii[i] = ii X_ii[i] = ii
return X_ii return X_ii
@njit('uint32(uint32[:, :], int16, int16, int16, int16)')
def __compute_feature__(ii: np.ndarray, x: int, y: int, w: int, h: int) -> int:
"""Compute a feature on an integrated image at a specific coordinate (CPU version).
Args:
ii (np.ndarray): Integrated image.
x (int): X coordinate.
y (int): Y coordinate.
w (int): width of the feature.
h (int): height of the feature.
Returns:
int: Computed feature.
"""
return ii[y + h, x + w] + ii[y, x] - ii[y + h, x] - ii[y, x + w]
@njit('int32[:, :](uint8[:, :, :, :], uint32[:, :, :])')
def apply_features(feats: np.ndarray, X_ii: np.ndarray) -> np.ndarray:
"""Apply the features on a integrated image dataset (CPU version).
Args:
feats (np.ndarray): Features to apply.
X_ii (np.ndarray): Integrated image dataset.
Returns:
np.ndarray: Applied features.
"""
X_feat = np.empty((feats.shape[0], X_ii.shape[0]), dtype = np.int32)
for i, (p, n) in enumerate(tqdm_iter(feats, "Applying features")):
for j, x_i in enumerate(X_ii):
p_x, p_y, p_w, p_h = p[0]
p1_x, p1_y, p1_w, p1_h = p[1]
n_x, n_y, n_w, n_h = n[0]
n1_x, n1_y, n1_w, n1_h = n[1]
p1 = __compute_feature__(x_i, p_x, p_y, p_w, p_h) + __compute_feature__(x_i, p1_x, p1_y, p1_w, p1_h)
n1 = __compute_feature__(x_i, n_x, n_y, n_w, n_h) + __compute_feature__(x_i, n1_x, n1_y, n1_w, n1_h)
X_feat[i, j] = int32(p1) - int32(n1)
return X_feat
@njit('int32[:, :](int32[:, :], uint16[:, :], uint8[:], float64[:])') @njit('int32[:, :](int32[:, :], uint16[:, :], uint8[:], float64[:])')
def train_weak_clf(X_feat: np.ndarray, X_feat_argsort: np.ndarray, y: np.ndarray, weights: np.ndarray) -> np.ndarray: def train_weak_clf(X_feat: np.ndarray, X_feat_argsort: np.ndarray, y: np.ndarray, weights: np.ndarray) -> np.ndarray:
"""Train the weak classifiers on a given dataset (CPU version). """Train the weak classifiers on a given dataset (CPU version).
Args: Args:
X_feat (np.ndarray): Feature images dataset. X_feat (np.ndarray): Feature images dataset
X_feat_argsort (np.ndarray): Sorted indexes of the integrated features. X_feat_argsort (np.ndarray): Sorted indexes of the integrated features
y (np.ndarray): Labels of the features. y (np.ndarray): Labels of the features
weights (np.ndarray): Weights of the features. weights (np.ndarray): Weights of the features
Returns: Returns:
np.ndarray: Trained weak classifiers. np.ndarray: Trained weak classifiers
""" """
total_pos, total_neg = weights[y == 1].sum(), weights[y == 0].sum() total_pos, total_neg = weights[y == 1].sum(), weights[y == 0].sum()
@ -112,29 +71,85 @@ def train_weak_clf(X_feat: np.ndarray, X_feat_argsort: np.ndarray, y: np.ndarray
classifiers[i] = (best_threshold, best_polarity) classifiers[i] = (best_threshold, best_polarity)
return classifiers return classifiers
@njit('uint32(uint32[:, :], int16, int16, int16, int16)')
def __compute_feature__(ii: np.ndarray, x: int, y: int, w: int, h: int) -> int:
"""Compute a feature on an integrated image at a specific coordinate (CPU version).
Args:
ii (np.ndarray): Integrated image
x (int): X coordinate
y (int): Y coordinate
w (int): width of the feature
h (int): height of the feature
Returns:
int: Computed feature
"""
return ii[y + h, x + w] + ii[y, x] - ii[y + h, x] - ii[y, x + w]
@njit('int32[:, :](uint8[:, :, :, :], uint32[:, :, :])')
def apply_features(feats: np.ndarray, X_ii: np.ndarray) -> np.ndarray:
"""Apply the features on a integrated image dataset (CPU version).
Args:
feats (np.ndarray): Features to apply
X_ii (np.ndarray): Integrated image dataset
Returns:
np.ndarray: Applied features
"""
X_feat = np.empty((feats.shape[0], X_ii.shape[0]), dtype = np.int32)
for i, (p, n) in enumerate(tqdm_iter(feats, "Applying features")):
for j, x_i in enumerate(X_ii):
p_x, p_y, p_w, p_h = p[0]
p1_x, p1_y, p1_w, p1_h = p[1]
n_x, n_y, n_w, n_h = n[0]
n1_x, n1_y, n1_w, n1_h = n[1]
p1 = __compute_feature__(x_i, p_x, p_y, p_w, p_h) + __compute_feature__(x_i, p1_x, p1_y, p1_w, p1_h)
n1 = __compute_feature__(x_i, n_x, n_y, n_w, n_h) + __compute_feature__(x_i, n1_x, n1_y, n1_w, n1_h)
X_feat[i, j] = int32(p1) - int32(n1)
return X_feat
@njit('int32(int32[:], uint16[:], int32, int32)') @njit('int32(int32[:], uint16[:], int32, int32)')
def as_partition(a: np.ndarray, indices: np.ndarray, l: int, h: int) -> int: def _as_partition_(d_a: np.ndarray, d_indices: np.ndarray, low: int, high: int) -> int:
i = l - 1 """Partition of the argsort algorithm.
j = l
for j in range(l, h + 1): Args:
if a[indices[j]] < a[indices[h]]: d_a (np.ndarray): Array on device to sort
d_indices (np.ndarray): Array of indices on device to write to
low (int): lower bound to sort
high (int): higher bound to sort
Returns:
int: Last index sorted
"""
i, j = low - 1, low
for j in range(low, high + 1):
if d_a[d_indices[j]] < d_a[d_indices[high]]:
i += 1 i += 1
indices[i], indices[j] = indices[j], indices[i] d_indices[i], d_indices[j] = d_indices[j], d_indices[i]
i += 1 i += 1
indices[i], indices[j] = indices[j], indices[i] d_indices[i], d_indices[j] = d_indices[j], d_indices[i]
return i return i
@njit('void(int32[:], uint16[:], int32, int32)') @njit('void(int32[:], uint16[:], int32, int32)')
def argsort_bounded(a: np.ndarray, indices: np.ndarray, l: int, h: int): def argsort_bounded(d_a: np.ndarray, d_indices: np.ndarray, low: int, high: int) -> None:
total = h - l + 1; """Perform an indirect sort of a given array within a given bound.
stack = np.empty((total,), dtype = np.int32)
stack[0] = l
stack[1] = h
top = 1;
low = l Args:
high = h d_a (np.ndarray): Array to sort
d_indices (np.ndarray): Array of indices to write to
low (int): lower bound to sort
high (int): higher bound to sort
"""
total = high - low + 1
stack = np.empty((total,), dtype = np.int32)
stack[0] = low
stack[1] = high
top = 1
while top >= 0: while top >= 0:
high = stack[top] high = stack[top]
@ -143,24 +158,32 @@ def argsort_bounded(a: np.ndarray, indices: np.ndarray, l: int, h: int):
top -= 1 top -= 1
if low >= high: if low >= high:
break; break
p = as_partition(a, indices, low, high); p = _as_partition_(d_a, d_indices, low, high)
if p - 1 > low: if p - 1 > low:
top += 1 top += 1
stack[top] = low; stack[top] = low
top += 1 top += 1
stack[top] = p - 1; stack[top] = p - 1
if p + 1 < high: if p + 1 < high:
top += 1 top += 1
stack[top] = p + 1; stack[top] = p + 1
top += 1 top += 1
stack[top] = high; stack[top] = high
@njit('uint16[:, :](int32[:, :])') @njit('uint16[:, :](int32[:, :])')
def argsort(X_feat: np.ndarray) -> np.ndarray: def argsort(X_feat: np.ndarray) -> np.ndarray:
"""Perform an indirect sort of a given array.
Args:
X_feat (np.ndarray): Array to sort
Returns:
np.ndarray: Array of indices that sort the array
"""
indices = np.empty_like(X_feat, dtype = np.uint16) indices = np.empty_like(X_feat, dtype = np.uint16)
indices[:, :] = np.arange(indices.shape[1]) indices[:, :] = np.arange(indices.shape[1])
for i in tqdm_iter(range(X_feat.shape[0]), "argsort"): for i in tqdm_iter(range(X_feat.shape[0]), "argsort"):

179
python/ViolaJonesGPU.py
View File

@ -12,10 +12,10 @@ def __scanCPU_3d__(X: np.ndarray) -> np.ndarray:
"""Prefix Sum (scan) of a given dataset. """Prefix Sum (scan) of a given dataset.
Args: Args:
X (np.ndarray): Dataset of images to apply sum. X (np.ndarray): Dataset of images to apply sum
Returns: Returns:
np.ndarray: Scanned dataset of images. np.ndarray: Scanned dataset of images
""" """
for x in range(X.shape[0]): for x in range(X.shape[0]):
for y in range(X.shape[1]): for y in range(X.shape[1]):
@ -30,10 +30,10 @@ def __kernel_scan_3d__(n: int, j: int, d_inter: np.ndarray, d_a: np.ndarray) ->
"""GPU kernel used to do a parallel prefix sum (scan). """GPU kernel used to do a parallel prefix sum (scan).
Args: Args:
n (int): n (int): Number of width blocks
j (int): [description] j (int): Temporary sum index
d_inter (np.ndarray): [description] d_inter (np.ndarray): Temporary sums in device to add
d_a (np.ndarray): [description] d_a (np.ndarray): Dataset of images in device to apply sum
""" """
x_coor, y_coor = cuda.grid(2) x_coor, y_coor = cuda.grid(2)
@ -76,10 +76,10 @@ def __add_3d__(d_X: np.ndarray, d_s: np.ndarray, n: int, m: int) -> None:
"""GPU kernel for parallel sum. """GPU kernel for parallel sum.
Args: Args:
d_X (np.ndarray): Dataset of images. d_X (np.ndarray): Dataset of images in device
d_s (np.ndarray): Temporary sums to add. d_s (np.ndarray): Temporary sums in device to add
n (int): Number of width blocks. n (int): Number of width blocks
m (int): Height of a block. m (int): Height of a block
""" """
x_coor, y_coor = cuda.grid(2) x_coor, y_coor = cuda.grid(2)
if x_coor < n and y_coor < m: if x_coor < n and y_coor < m:
@ -91,10 +91,10 @@ def __scanGPU_3d__(X: np.ndarray) -> np.ndarray:
Read more: https://developer.nvidia.com/gpugems/gpugems3/part-vi-gpu-computing/chapter-39-parallel-prefix-sum-scan-cuda Read more: https://developer.nvidia.com/gpugems/gpugems3/part-vi-gpu-computing/chapter-39-parallel-prefix-sum-scan-cuda
Args: Args:
X (np.ndarray): Dataset of images. X (np.ndarray): Dataset of images
Returns: Returns:
np.ndarray: Scanned dataset of images. np.ndarray: Scanned dataset of images
""" """
k, height, n = X.shape k, height, n = X.shape
n_block_x, n_block_y = np.ceil(np.divide(X.shape[1:], NB_THREADS_2D)).astype(np.uint64) n_block_x, n_block_y = np.ceil(np.divide(X.shape[1:], NB_THREADS_2D)).astype(np.uint64)
@ -131,10 +131,10 @@ def __transpose_kernel__(d_X: np.ndarray, d_Xt: np.ndarray) -> None:
"""GPU kernel of the function __transpose_3d__. """GPU kernel of the function __transpose_3d__.
Args: Args:
d_X (np.ndarray): Dataset of images. d_X (np.ndarray): Dataset of images in device
d_Xt(np.ndarray): Transposed dataset of images. d_Xt(np.ndarray): Transposed dataset of images
width (int): Width of each images in the dataset. width (int): Width of each images in the dataset
height (int): Height of each images in the dataset. height (int): Height of each images in the dataset
""" """
temp = cuda.shared.array(NB_THREADS_2D, dtype = uint32) temp = cuda.shared.array(NB_THREADS_2D, dtype = uint32)
@ -152,10 +152,10 @@ def __transpose_3d__(X: np.ndarray) -> np.ndarray:
"""Transpose every images in the given dataset. """Transpose every images in the given dataset.
Args: Args:
X (np.ndarray): Dataset of images. X (np.ndarray): Dataset of images
Returns: Returns:
np.ndarray: Transposed dataset of images. np.ndarray: Transposed dataset of images
""" """
n_block_x, n_block_z = np.ceil(np.divide(X.shape[1:], NB_THREADS_2D)).astype(np.uint64) n_block_x, n_block_z = np.ceil(np.divide(X.shape[1:], NB_THREADS_2D)).astype(np.uint64)
d_X = cuda.to_device(X) d_X = cuda.to_device(X)
@ -167,10 +167,10 @@ def set_integral_image(X: np.ndarray) -> np.ndarray:
"""Transform the input images in integrated images (GPU version). """Transform the input images in integrated images (GPU version).
Args: Args:
X (np.ndarray): Dataset of images. X (np.ndarray): Dataset of images
Returns: Returns:
np.ndarray: Dataset of integrated images. np.ndarray: Dataset of integrated images
""" """
X = X.astype(np.uint32) X = X.astype(np.uint32)
X = __scanGPU_3d__(X) X = __scanGPU_3d__(X)
@ -184,13 +184,13 @@ def __train_weak_clf_kernel__(d_classifiers: np.ndarray, d_y: np.ndarray, d_X_fe
"""GPU kernel of the function train_weak_clf. """GPU kernel of the function train_weak_clf.
Args: Args:
d_classifiers (np.ndarray): Weak classifiers to train. d_classifiers (np.ndarray): Weak classifiers to train
d_y (np.ndarray): Labels of the features. d_y (np.ndarray): Labels of the features
d_X_feat (np.ndarray): Feature images dataset. d_X_feat (np.ndarray): Feature images dataset
d_X_feat_argsort (np.ndarray): Sorted indexes of the integrated features. d_X_feat_argsort (np.ndarray): Sorted indexes of the integrated features
d_weights (np.ndarray): Weights of the features. d_weights (np.ndarray): Weights of the features
total_pos (float): Total of positive labels in the dataset. total_pos (float): Total of positive labels in the dataset
total_neg (float): Total of negative labels in the dataset. total_neg (float): Total of negative labels in the dataset
""" """
i = cuda.blockIdx.x * cuda.blockDim.x * cuda.blockDim.y * cuda.blockDim.z i = cuda.blockIdx.x * cuda.blockDim.x * cuda.blockDim.y * cuda.blockDim.z
i += cuda.threadIdx.x * cuda.blockDim.y * cuda.blockDim.z i += cuda.threadIdx.x * cuda.blockDim.y * cuda.blockDim.z
@ -224,13 +224,13 @@ def train_weak_clf(X_feat: np.ndarray, X_feat_argsort: np.ndarray, y: np.ndarray
"""Train the weak classifiers on a given dataset (GPU version). """Train the weak classifiers on a given dataset (GPU version).
Args: Args:
X_feat (np.ndarray): Feature images dataset. X_feat (np.ndarray): Feature images dataset
X_feat_argsort (np.ndarray): Sorted indexes of the integrated features. X_feat_argsort (np.ndarray): Sorted indexes of the integrated features
y (np.ndarray): Labels of the features. y (np.ndarray): Labels of the features
weights (np.ndarray): Weights of the features. weights (np.ndarray): Weights of the features
Returns: Returns:
np.ndarray: Trained weak classifiers. np.ndarray: Trained weak classifiers
""" """
total_pos, total_neg = weights[y == 1].sum(), weights[y == 0].sum() total_pos, total_neg = weights[y == 1].sum(), weights[y == 0].sum()
d_classifiers = cuda.to_device(np.empty((X_feat.shape[0], 2), dtype = np.int32)) d_classifiers = cuda.to_device(np.empty((X_feat.shape[0], 2), dtype = np.int32))
@ -247,14 +247,14 @@ def __compute_feature__(ii: np.ndarray, x: int, y: int, w: int, h: int) -> int:
"""Compute a feature on an integrated image at a specific coordinate (GPU version). """Compute a feature on an integrated image at a specific coordinate (GPU version).
Args: Args:
ii (np.ndarray): Integrated image. ii (np.ndarray): Integrated image
x (int): X coordinate. x (int): X coordinate
y (int): Y coordinate. y (int): Y coordinate
w (int): width of the feature. w (int): width of the feature
h (int): height of the feature. h (int): height of the feature
Returns: Returns:
int: Computed feature. int: Computed feature
""" """
return ii[y + h, x + w] + ii[y, x] - ii[y + h, x] - ii[y, x + w] return ii[y + h, x + w] + ii[y, x] - ii[y + h, x] - ii[y, x + w]
@ -263,11 +263,11 @@ def __apply_feature_kernel__(X_feat: np.ndarray, feats: np.ndarray, X_ii: np.nda
"""GPU kernel of the function apply_features. """GPU kernel of the function apply_features.
Args: Args:
X_feat (np.ndarray): Feature images dataset. X_feat (np.ndarray): Feature images dataset on device
feats (np.ndarray): Features to apply. feats (np.ndarray): Features on device to apply
X_ii (np.ndarray): Integrated image dataset. X_ii (np.ndarray): Integrated image dataset on device
n (int): Number of features. n (int): Number of features
m (int): Number of images of the dataset. m (int): Number of images of the dataset
""" """
x, y = cuda.grid(2) x, y = cuda.grid(2)
if x >= feats.shape[0] or y >= X_ii.shape[0]: if x >= feats.shape[0] or y >= X_ii.shape[0]:
@ -288,11 +288,11 @@ def apply_features(feats: np.ndarray, X_ii: np.ndarray) -> np.ndarray:
"""Apply the features on a integrated image dataset (GPU version). """Apply the features on a integrated image dataset (GPU version).
Args: Args:
feats (np.ndarray): Features to apply. feats (np.ndarray): Features to apply
X_ii (np.ndarray): Integrated image dataset. X_ii (np.ndarray): Integrated image dataset
Returns: Returns:
np.ndarray: Applied features. np.ndarray: Applied features
""" """
d_X_feat = cuda.to_device(np.empty((feats.shape[0], X_ii.shape[0]), dtype = np.int32)) d_X_feat = cuda.to_device(np.empty((feats.shape[0], X_ii.shape[0]), dtype = np.int32))
d_feats = cuda.to_device(feats) d_feats = cuda.to_device(feats)
@ -303,28 +303,44 @@ def apply_features(feats: np.ndarray, X_ii: np.ndarray) -> np.ndarray:
return d_X_feat.copy_to_host() return d_X_feat.copy_to_host()
@cuda.jit('int32(int32[:], uint16[:], int32, int32)', device = True) @cuda.jit('int32(int32[:], uint16[:], int32, int32)', device = True)
def as_partition(a: np.ndarray, indices: np.ndarray, l: int, h: int) -> int: def _as_partition_(d_a: np.ndarray, d_indices: np.ndarray, l: int, h: int) -> int:
"""Partition of the argsort algorithm.
Args:
d_a (np.ndarray): Array on device to sort
d_indices (np.ndarray): Array of indices on device to write to
low (int): lower bound to sort
high (int): higher bound to sort
Returns:
int: Last index sorted
"""
i = l - 1 i = l - 1
j = l j = l
for j in range(l, h + 1): for j in range(l, h + 1):
if a[indices[j]] < a[indices[h]]: if d_a[d_indices[j]] < d_a[d_indices[h]]:
i += 1 i += 1
indices[i], indices[j] = indices[j], indices[i] d_indices[i], d_indices[j] = d_indices[j], d_indices[i]
i += 1 i += 1
indices[i], indices[j] = indices[j], indices[i] d_indices[i], d_indices[j] = d_indices[j], d_indices[i]
return i return i
@cuda.jit('void(int32[:], uint16[:], int32, int32)', device = True) @cuda.jit('void(int32[:], uint16[:], int32, int32)', device = True)
def argsort_bounded(a: np.ndarray, indices: np.ndarray, l: int, h: int) -> None: def argsort_bounded(d_a: np.ndarray, d_indices: np.ndarray, low: int, high: int) -> None:
#total = h - l + 1; """Perform an indirect sort of a given array within a given bound.
stack = cuda.local.array(6977, int32)
stack[0] = l
stack[1] = h
top = 1;
low = l Args:
high = h d_a (np.ndarray): Array on device to sort
d_indices (np.ndarray): Array of indices on device to write to
low (int): lower bound to sort
high (int): higher bound to sort
"""
#total = high - low + 1;
stack = cuda.local.array(6977, int32)
stack[0] = low
stack[1] = high
top = 1
while top >= 0: while top >= 0:
high = stack[top] high = stack[top]
@ -333,34 +349,49 @@ def argsort_bounded(a: np.ndarray, indices: np.ndarray, l: int, h: int) -> None:
top -= 1 top -= 1
if low >= high: if low >= high:
break; break
p = as_partition(a, indices, low, high); p = _as_partition_(d_a, d_indices, low, high)
if p - 1 > low: if p - 1 > low:
top += 1 top += 1
stack[top] = low; stack[top] = low
top += 1 top += 1
stack[top] = p - 1; stack[top] = p - 1
if p + 1 < high: if p + 1 < high:
top += 1 top += 1
stack[top] = p + 1; stack[top] = p + 1
top += 1 top += 1
stack[top] = high; stack[top] = high
@cuda.jit('void(int32[:, :], uint16[:, :])') @cuda.jit('void(int32[:, :], uint16[:, :])')
def argsort_flatter(X_feat: np.ndarray, indices: np.ndarray) -> None: def argsort_flatter(d_a: np.ndarray, d_indices: np.ndarray) -> None:
i = cuda.blockIdx.x * cuda.blockDim.x + cuda.threadIdx.x # TODO Finish doxygen
if i < X_feat.shape[0]: """Cuda kernel where argsort is applied to every columns of a given 2D array.
for j in range(indices.shape[1]):
indices[i, j] = j
argsort_bounded(X_feat[i], indices[i], 0, X_feat.shape[1] - 1)
def argsort(X_feat: np.ndarray) -> np.ndarray: Args:
indices = np.empty_like(X_feat, dtype = np.uint16) d_a (np.ndarray): Array in device to sort
n_blocks = int(np.ceil(np.divide(X_feat.shape[0], NB_THREADS))) d_indices (np.ndarray): Array of indices on device to write to
d_X_feat = cuda.to_device(X_feat) """
i = cuda.blockIdx.x * cuda.blockDim.x + cuda.threadIdx.x
if i < d_a.shape[0]:
for j in range(d_indices.shape[1]):
d_indices[i, j] = j
argsort_bounded(d_a[i], d_indices[i], 0, d_a.shape[1] - 1)
def argsort(a: np.ndarray) -> np.ndarray:
"""Perform an indirect sort of a given array
Args:
a (np.ndarray): Array to sort
Returns:
np.ndarray: Array of indices that sort the array
"""
indices = np.empty_like(a, dtype = np.uint16)
n_blocks = int(np.ceil(np.divide(a.shape[0], NB_THREADS)))
d_X_feat = cuda.to_device(a)
d_indices = cuda.to_device(indices) d_indices = cuda.to_device(indices)
argsort_flatter[n_blocks, NB_THREADS](d_X_feat, d_indices) argsort_flatter[n_blocks, NB_THREADS](d_X_feat, d_indices)
cuda.synchronize() cuda.synchronize()

11
python/activate.sh
View File

@ -3,9 +3,8 @@
# Exit if any of the command doesn't exit with code 0 # Exit if any of the command doesn't exit with code 0
set -e set -e
EXEC_DIR=$1 test -z "$EXEC_DIR" && EXEC_DIR=.
test -z "$EXEC_DIR" && EXEC_DIR=.. test -z "$VENV_PATH" && VENV_PATH="$EXEC_DIR/venv"
VENV_PATH=$EXEC_DIR/python/venv
activate(){ activate(){
if [ ! -d "$VENV_PATH" ]; then if [ ! -d "$VENV_PATH" ]; then
@ -16,9 +15,9 @@ activate(){
echo 'Updating base pip packages' echo 'Updating base pip packages'
python -m pip install -U setuptools pip python -m pip install -U setuptools pip
echo 'Installing requirements' echo 'Installing requirements'
pip install -r "$EXEC_DIR"/python/requirements.txt pip install -r requirements.txt
elif [ -f "$VENV_PATH"/Scripts/activate ]; then source "$VENV_PATH"/Scripts/activate elif [ -f "$VENV_PATH"/Scripts/activate ]; then . "$VENV_PATH"/Scripts/activate
elif [ -f "$VENV_PATH"/bin/activate ]; then source "$VENV_PATH"/bin/activate elif [ -f "$VENV_PATH"/bin/activate ]; then . "$VENV_PATH"/bin/activate
else else
echo 'Python virtual environnement not detected' echo 'Python virtual environnement not detected'
exit 1 exit 1

92
python/common.py
View File

@ -1,29 +1,29 @@
from toolbox import picke_multi_loader, format_time_ns, unit_test_argsort_2d from toolbox import pickle_multi_loader, format_time_ns, unit_test_argsort_2d, header, footer, formatted_line, formatted_row
from typing import List, Tuple from typing import List, Tuple
from time import perf_counter_ns from time import perf_counter_ns
from sys import stderr
import numpy as np import numpy as np
from config import OUT_DIR, DATA_DIR, __DEBUG from config import OUT_DIR, DATA_DIR, __DEBUG
def unit_test(TS: List[int], labels: List[str] = ["CPU", "GPU", "PY", "PGPU"], tol: float = 1e-8) -> None: def unit_test(TS: List[int], labels: List[str] = ['CPU', 'GPU', 'PY', 'PGPU'], tol: float = 1e-8) -> None:
"""Test if the each result is equals to other devices. """Test if the each result is equals to other devices.
Given ViolaJones is a deterministic algorithm, the results no matter the device should be the same Given ViolaJones is a fully deterministic algorithm. The results, regardless the device, should be the same
(given the floating point fluctuations), this function check this assertion. (given the floating point fluctuations), this function check this assertion.
Args: Args:
TS (List[int]): Number of trained weak classifiers. TS (List[int]): Number of trained weak classifiers
labels (List[str], optional): List of the trained device names. Defaults to ["CPU", "GPU", "PY", "PGPU"] (see config.py for more info). labels (List[str], optional): List of the trained device names. Defaults to ['CPU', 'GPU', 'PY', 'PGPU'] (see config.py for more info)
tol (float, optional): Float difference tolerance. Defaults to 1e-8. tol (float, optional): Float difference tolerance. Defaults to 1e-8
""" """
if len(labels) < 2: if len(labels) < 2:
return print("Not enough devices to test") return print('Not enough devices to test')
print(f"\n| {'Unit testing':<37} | {'Test state':<10} | {'Time spent (ns)':<18} | {'Formatted time spent':<29} |") unit_gaps = [37, -10, -18, 29]
print(f"|{'-'*39}|{'-'*12}|{'-'*20}|{'-'*31}|") header(unit_gaps, ['Unit testing', 'Test state', 'Time spent (ns)', 'Formatted time spent'])
fnc_s = perf_counter_ns() unit_timestamp = perf_counter_ns()
n_total = 0 n_total, n_success = 0, 0
n_success = 0
def test_fnc(title, fnc): def test_fnc(title, fnc):
nonlocal n_total, n_success nonlocal n_total, n_success
@ -32,96 +32,102 @@ def unit_test(TS: List[int], labels: List[str] = ["CPU", "GPU", "PY", "PGPU"], t
state = fnc() state = fnc()
e = perf_counter_ns() - s e = perf_counter_ns() - s
if state: if state:
print(f"| {title:<37} | {'Passed':>10} | {e:>18,} | {format_time_ns(e):<29} |") formatted_row(unit_gaps, [title, 'Passed', f'{e:,}', format_time_ns(e)])
n_success += 1 n_success += 1
else: else:
print(f"| {title:<37} | {'Failed':>10} | {e:>18,} | {format_time_ns(e):<29} |") formatted_row(unit_gaps, [title, 'Failed', f'{e:,}', format_time_ns(e)])
for set_name in ["train", "test"]: for set_name in ['train', 'test']:
for filename in ["ii", "feat"]: for filename in ['ii', 'feat']:
title = f"X_{set_name}_{filename}" title = f'X_{set_name}_{filename}'
print(f"{filename}...", end = "\r") print(f'{filename}...', file = stderr, end = '\r')
bs = picke_multi_loader([f"{title}_{label}" for label in labels], OUT_DIR) bs = pickle_multi_loader([f'{title}_{label}' for label in labels], OUT_DIR)
for i, (b1, l1) in enumerate(zip(bs, labels)): for i, (b1, l1) in enumerate(zip(bs, labels)):
if b1 is None: if b1 is None:
if __DEBUG: if __DEBUG:
print(f"| {title:<22} - {l1:<12} | {'Skipped':>10} | {'None':>18} | {'None':<29} |") formatted_row(unit_gaps, [f'{title:<22} - {l1:<12}', 'Skipped', 'None', 'None'])
continue continue
for j, (b2, l2) in enumerate(zip(bs, labels)): for j, (b2, l2) in enumerate(zip(bs, labels)):
if i >= j: if i >= j:
continue continue
if b2 is None: if b2 is None:
if __DEBUG: if __DEBUG:
print(f"| {title:<22} - {l1:<4} vs {l2:<4} | {'Skipped':>10} | {'None':>18} | {'None':<29} |") formatted_row(unit_gaps, [f'{title:<22} - {l1:<4} vs {l2:<4}', 'Skipped', 'None', 'None'])
continue continue
test_fnc(f"{title:<22} - {l1:<4} vs {l2:<4}", lambda: np.abs(b1 - b2).mean() < tol) test_fnc(f'{title:<22} - {l1:<4} vs {l2:<4}', lambda: np.abs(b1 - b2).mean() < tol)
title = f"X_{set_name}_feat_argsort" title = f'X_{set_name}_feat_argsort'
print(f"Loading {title}...", end = "\r") print(f'Loading {title}...', file = stderr, end = '\r')
feat = None feat = None
bs = [] bs = []
for label in labels: for label in labels:
if feat is None: if feat is None:
feat_tmp = picke_multi_loader([f"X_{set_name}_feat_{label}"], OUT_DIR)[0] feat_tmp = pickle_multi_loader([f'X_{set_name}_feat_{label}'], OUT_DIR)[0]
if feat_tmp is not None: if feat_tmp is not None:
feat = feat_tmp feat = feat_tmp
bs.append(picke_multi_loader([f"{title}_{label}"], OUT_DIR)[0]) bs.append(pickle_multi_loader([f'{title}_{label}'], OUT_DIR)[0])
for i, (b1, l1) in enumerate(zip(bs, labels)): for i, (b1, l1) in enumerate(zip(bs, labels)):
if b1 is None: if b1 is None:
if __DEBUG: if __DEBUG:
print(f"| {title:<22} - {l1:<12} | {'Skipped':>10} | {'None':>18} | {'None':<29} |") formatted_row(unit_gaps, [f'{title:<22} - {l1:<12}', 'Skipped', 'None', 'None'])
continue continue
if feat is not None: if feat is not None:
test_fnc(f"{title:<22} - {l1:<4} argsort", lambda: unit_test_argsort_2d(feat, b1)) test_fnc(f'{title:<22} - {l1:<4} argsort', lambda: unit_test_argsort_2d(feat, b1))
for j, (b2, l2) in enumerate(zip(bs, labels)): for j, (b2, l2) in enumerate(zip(bs, labels)):
if i >= j: if i >= j:
continue continue
if b2 is None: if b2 is None:
if __DEBUG: if __DEBUG:
print(f"| {title:<22} - {l1:<4} vs {l2:<4} | {'Skipped':>10} | {'None':>18} | {'None':<29} |") formatted_row(unit_gaps, [f'{title:<22} - {l1:<4} vs {l2:<4}', 'Skipped', 'None', 'None'])
continue continue
test_fnc(f"{title:<22} - {l1:<4} vs {l2:<4}", lambda: np.abs(b1 - b2).mean() < tol) test_fnc(f'{title:<22} - {l1:<4} vs {l2:<4}', lambda: np.abs(b1 - b2).mean() < tol)
for T in TS: for T in TS:
for filename in ["alphas", "final_classifiers"]: for filename in ['alphas', 'final_classifiers']:
print(f"{filename}_{T}...", end = "\r") print(f'{filename}_{T}...', file = stderr, end = '\r')
bs = picke_multi_loader([f"{filename}_{T}_{label}" for label in labels]) bs = pickle_multi_loader([f'{filename}_{T}_{label}' for label in labels])
for i, (b1, l1) in enumerate(zip(bs, labels)): for i, (b1, l1) in enumerate(zip(bs, labels)):
if b1 is None: if b1 is None:
if __DEBUG: if __DEBUG:
print(f"| {filename + '_' + str(T):<22} - {l1:<12} | {'Skipped':>10} | {'None':>18} | {'None':<29} |") formatted_row(unit_gaps, [f"{filename + '_' + str(T):<22} - {l1:<12}", 'Skipped', 'None', 'None'])
continue continue
for j, (b2, l2) in enumerate(zip(bs, labels)): for j, (b2, l2) in enumerate(zip(bs, labels)):
if i >= j: if i >= j:
continue continue
if b2 is None: if b2 is None:
if __DEBUG: if __DEBUG:
print(f"| {filename + '_' + str(T):<22} - {l1:<4} vs {l2:<4} | {'Skipped':>10} | {'None':>18} | {'None':<29} |") formatted_row(unit_gaps, [f"{filename + '_' + str(T):<22} - {l1:<4} vs {l2:<4}", 'Skipped', 'None', 'None'])
continue continue
test_fnc(f"{filename + '_' + str(T):<22} - {l1:<4} vs {l2:<4}", lambda: np.abs(b1 - b2).mean() < tol) test_fnc(f"{filename + '_' + str(T):<22} - {l1:<4} vs {l2:<4}", lambda: np.abs(b1 - b2).mean() < tol)
print(f"|{'-'*39}|{'-'*12}|{'-'*20}|{'-'*31}|") time_spent = perf_counter_ns() - unit_timestamp
e = perf_counter_ns() - fnc_s
print(f"| {'Unit testing summary':<37} | {str(n_success) + '/' + str(n_total):>10} | {e:>18,} | {format_time_ns(e):<29} |") if n_total == 0:
formatted_row(unit_gaps, ['Unit testing summary', 'No files', f'{time_spent:,}', format_time_ns(time_spent)])
else:
formatted_line(unit_gaps, '├', '┼', '─', '┤')
formatted_row(unit_gaps, ['Unit testing summary', f'{n_success}/{n_total}', f'{time_spent:,}', format_time_ns(time_spent)])
footer(unit_gaps)
def load_datasets(data_dir: str = DATA_DIR) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: def load_datasets(data_dir: str = DATA_DIR) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
"""Load the datasets. """Load the datasets.
Args: Args:
data_dir (str, optional): [description]. Defaults to DATA_DIR (see config.py). data_dir (str, optional): [description]. Defaults to DATA_DIR (see config.py)
Returns: Returns:
Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: [description] Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: X_train, y_train, X_test, y_test
""" """
bytes_to_int_list = lambda b: list(map(int, b.rstrip().split(" "))) bytes_to_int_list = lambda b: list(map(int, b.rstrip().split(' ')))
def load(set_name: str) -> np.ndarray: def load(set_name: str) -> np.ndarray:
with open(f"{data_dir}/{set_name}.bin", "r") as f: with open(f'{data_dir}/{set_name}.bin', 'r') as f:
shape = bytes_to_int_list(f.readline()) shape = bytes_to_int_list(f.readline())
return np.asarray(bytes_to_int_list(f.readline()), dtype = np.uint8).reshape(shape) return np.asarray(bytes_to_int_list(f.readline()), dtype = np.uint8).reshape(shape)
return load("X_train"), load("y_train"), load("X_test"), load("y_test") return load('X_train'), load('y_train'), load('X_test'), load('y_test')

7
python/convert_dataset.py
View File

@ -5,6 +5,9 @@ from sys import argv
import numpy as np import numpy as np
from os import path, listdir from os import path, listdir
# Induce determinism
np.random.seed(133742)
# Makes the "leave" argument default to False # Makes the "leave" argument default to False
tqdm = partial(tqdm, leave = False) tqdm = partial(tqdm, leave = False)
@ -42,8 +45,8 @@ def __main__(data_path: str) -> None:
y.append(y_i) y.append(y_i)
X, y = np.asarray(X), np.asarray(y) X, y = np.asarray(X), np.asarray(y)
# idx = np.random.permutation(y.shape[0]) idx = np.random.permutation(y.shape[0])
# X, y = X[idx], y[idx] X, y = X[idx], y[idx]
for org, s in tqdm(zip("Xy", [X, y]), desc = f"Writing {set_name}"): for org, s in tqdm(zip("Xy", [X, y]), desc = f"Writing {set_name}"):
with open(f"{data_path}/{org}_{set_name}.bin", "w") as out: with open(f"{data_path}/{org}_{set_name}.bin", "w") as out:

17
python/decorators.py
View File

@ -2,6 +2,14 @@ from typing import Callable, Iterable, Union, Any
from tqdm import tqdm from tqdm import tqdm
def njit(f: Union[Callable, str] = None, *args, **kwargs) -> Callable: def njit(f: Union[Callable, str] = None, *args, **kwargs) -> Callable:
"""Wrapper for optional numba's njit decorator
Args:
f (Union[Callable, str], optional): Function to wrap with numba. Defaults to None.
Returns:
Callable: Wrapped function.
"""
def decorator(func: Callable) -> Any: def decorator(func: Callable) -> Any:
return func return func
@ -10,4 +18,13 @@ def njit(f: Union[Callable, str] = None, *args, **kwargs) -> Callable:
return decorator return decorator
def tqdm_iter(iter: Iterable, desc: str): def tqdm_iter(iter: Iterable, desc: str):
"""Wrapper for optional tqdm iterator progress bar.
Args:
iter (Iterable): Object to iterate over.
desc (str): Description written to stdout.
Returns:
_type_: Wrapped iterator.
"""
return tqdm(iter, leave = False, desc = desc) return tqdm(iter, leave = False, desc = desc)

104
python/projet.py
View File

@ -2,14 +2,15 @@
# Author: @saundersp # Author: @saundersp
from ViolaJones import train_viola_jones, classify_viola_jones from ViolaJones import train_viola_jones, classify_viola_jones
from toolbox import state_saver, picke_multi_loader, format_time_ns, benchmark_function, unit_test_argsort_2d #from toolbox import state_saver, pickle_multi_loader, format_time_ns, benchmark_function, unit_test_argsort_2d
from toolbox_unit_test import format_time_ns_test from toolbox import state_saver, format_time_ns, benchmark_function, unit_test_argsort_2d
from toolbox import header, footer, formatted_row, formatted_line from toolbox import header, footer, formatted_row, formatted_line
from toolbox_unit_test import format_time_test, format_time_ns_test
from sklearn.metrics import accuracy_score, f1_score, confusion_matrix from sklearn.metrics import accuracy_score, f1_score, confusion_matrix
from sklearn.feature_selection import SelectPercentile, f_classif #from sklearn.feature_selection import SelectPercentile, f_classif
from common import load_datasets, unit_test from common import load_datasets, unit_test
from ViolaJones import build_features, get_best_anova_features from ViolaJones import build_features # , get_best_anova_features
from typing import Tuple from typing import Tuple, List
from time import perf_counter_ns from time import perf_counter_ns
from os import makedirs from os import makedirs
import numpy as np import numpy as np
@ -28,35 +29,42 @@ else:
from ViolaJonesCPU import apply_features, set_integral_image, argsort from ViolaJonesCPU import apply_features, set_integral_image, argsort
label = 'CPU' if COMPILE_WITH_C else 'PY' label = 'CPU' if COMPILE_WITH_C else 'PY'
def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
"""Load the dataset, calculate features and integral images, apply features to images and calculate argsort of the featured images. """Execute the preprocessing phase
The preprocessing phase consist of the following steps :
- Load the dataset
- Calculate features
- Calculate integral images
- Apply features to images
- Calculate argsort of the featured images.
Returns: Returns:
Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: X_train_feat, X_train_feat_argsort, y_train, X_test_feat, y_test Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: Tuple containing in order : training features, training features sorted indexes, training labels, testing features, testing labels
""" """
# Creating state saver folders if they don't exist already # Creating state saver folders if they don't exist already
if SAVE_STATE: if SAVE_STATE:
for folder_name in ["models", "out"]: for folder_name in ['models', 'out']:
makedirs(folder_name, exist_ok = True) makedirs(folder_name, exist_ok = True)
preproc_timestamp = perf_counter_ns() preproc_timestamp = perf_counter_ns()
preproc_gaps = [49, -18, 29] preproc_gaps = [49, -18, 29]
header(['Preprocessing', 'Time spent (ns)', 'Formatted time spent'], preproc_gaps) header(preproc_gaps, ['Preprocessing', 'Time spent (ns)', 'Formatted time spent'])
X_train, y_train, X_test, y_test = state_saver('Loading sets', preproc_gaps[0], ['X_train', 'y_train', 'X_test', 'y_test'], X_train, y_train, X_test, y_test = state_saver('Loading sets', preproc_gaps[0], ['X_train', 'y_train', 'X_test', 'y_test'],
load_datasets, FORCE_REDO, SAVE_STATE) load_datasets, FORCE_REDO, SAVE_STATE)
if __DEBUG: if __DEBUG:
print("X_train") print('X_train')
print(X_train.shape) print(X_train.shape)
print(X_train[IDX_INSPECT]) print(X_train[IDX_INSPECT])
print("X_test") print('X_test')
print(X_test.shape) print(X_test.shape)
print(X_test[IDX_INSPECT]) print(X_test[IDX_INSPECT])
print("y_train") print('y_train')
print(y_train.shape) print(y_train.shape)
print(y_train[IDX_INSPECT: IDX_INSPECT + IDX_INSPECT_OFFSET]) print(y_train[IDX_INSPECT: IDX_INSPECT + IDX_INSPECT_OFFSET])
print("y_test") print('y_test')
print(y_test.shape) print(y_test.shape)
print(y_test[IDX_INSPECT: IDX_INSPECT + IDX_INSPECT_OFFSET]) print(y_test[IDX_INSPECT: IDX_INSPECT + IDX_INSPECT_OFFSET])
@ -64,7 +72,7 @@ def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
FORCE_REDO, SAVE_STATE) FORCE_REDO, SAVE_STATE)
if __DEBUG: if __DEBUG:
print("feats") print('feats')
print(feats.shape) print(feats.shape)
print(feats[IDX_INSPECT].ravel()) print(feats[IDX_INSPECT].ravel())
@ -74,10 +82,10 @@ def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
lambda: set_integral_image(X_test), FORCE_REDO, SAVE_STATE) lambda: set_integral_image(X_test), FORCE_REDO, SAVE_STATE)
if __DEBUG: if __DEBUG:
print("X_train_ii") print('X_train_ii')
print(X_train_ii.shape) print(X_train_ii.shape)
print(X_train_ii[IDX_INSPECT]) print(X_train_ii[IDX_INSPECT])
print("X_test_ii") print('X_test_ii')
print(X_test_ii.shape) print(X_test_ii.shape)
print(X_test_ii[IDX_INSPECT]) print(X_test_ii[IDX_INSPECT])
@ -88,25 +96,25 @@ def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
del X_train_ii, X_test_ii, feats del X_train_ii, X_test_ii, feats
if __DEBUG: if __DEBUG:
print("X_train_feat") print('X_train_feat')
print(X_train_feat.shape) print(X_train_feat.shape)
print(X_train_feat[IDX_INSPECT, : IDX_INSPECT_OFFSET]) print(X_train_feat[IDX_INSPECT, : IDX_INSPECT_OFFSET])
print("X_test_feat") print('X_test_feat')
print(X_test_feat.shape) print(X_test_feat.shape)
print(X_test_feat[IDX_INSPECT, : IDX_INSPECT_OFFSET]) print(X_test_feat[IDX_INSPECT, : IDX_INSPECT_OFFSET])
#indices = state_saver("Selecting best features training set", "indices", force_redo = True, save_state = SAVE_STATE, #indices = state_saver('Selecting best features training set', 'indices', force_redo = FORCE_REDO, save_state = SAVE_STATE,
# fnc = lambda: SelectPercentile(f_classif, percentile = 10).fit(X_train_feat.T, y_train).get_support(indices = True)) # fnc = lambda: SelectPercentile(f_classif, percentile = 10).fit(X_train_feat.T, y_train).get_support(indices = True))
#indices = state_saver("Selecting best features training set", "indices", force_redo = FORCE_REDO, save_state = SAVE_STATE, #indices = state_saver('Selecting best features training set', 'indices', force_redo = FORCE_REDO, save_state = SAVE_STATE,
# fnc = lambda: get_best_anova_features(X_train_feat, y_train)) # fnc = lambda: get_best_anova_features(X_train_feat, y_train))
#indices = benchmark_function("Selecting best features (manual)", lambda: get_best_anova_features(X_train_feat, y_train)) #indices = benchmark_function('Selecting best features (manual)', lambda: get_best_anova_features(X_train_feat, y_train))
#if __DEBUG: #if __DEBUG:
# print("indices") # print('indices')
# print(indices.shape) # print(indices.shape)
# print(indices[IDX_INSPECT: IDX_INSPECT + IDX_INSPECT_OFFSET]) # print(indices[IDX_INSPECT: IDX_INSPECT + IDX_INSPECT_OFFSET])
# assert indices.shape[0] == indices_new.shape[0], f"Indices length not equal : {indices.shape} != {indices_new.shape}" # assert indices.shape[0] == indices_new.shape[0], f'Indices length not equal : {indices.shape} != {indices_new.shape}'
# assert (eq := indices == indices_new).all(), f"Indices not equal : {eq.sum() / indices.shape[0]}" # assert (eq := indices == indices_new).all(), f'Indices not equal : {eq.sum() / indices.shape[0]}'
# X_train_feat, X_test_feat = X_train_feat[indices], X_test_feat[indices] # X_train_feat, X_test_feat = X_train_feat[indices], X_test_feat[indices]
@ -114,19 +122,20 @@ def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
lambda: argsort(X_train_feat), FORCE_REDO, SAVE_STATE) lambda: argsort(X_train_feat), FORCE_REDO, SAVE_STATE)
if __DEBUG: if __DEBUG:
print("X_train_feat_argsort") print('X_train_feat_argsort')
print(X_train_feat_argsort.shape) print(X_train_feat_argsort.shape)
print(X_train_feat_argsort[IDX_INSPECT, : IDX_INSPECT_OFFSET]) print(X_train_feat_argsort[IDX_INSPECT, : IDX_INSPECT_OFFSET])
benchmark_function('Arg unit test', preproc_gaps[0], lambda: unit_test_argsort_2d(X_train_feat, X_train_feat_argsort)) benchmark_function('Arg unit test', preproc_gaps[0], lambda: unit_test_argsort_2d(X_train_feat, X_train_feat_argsort))
X_test_feat_argsort = state_saver(f"Precalculating testing set argsort ({label})", f"X_test_feat_argsort_{label}", X_test_feat_argsort = state_saver(f'Precalculating testing set argsort ({label})', preproc_gaps[0], f'X_test_feat_argsort_{label}',
lambda: argsort(X_test_feat), FORCE_REDO, SAVE_STATE) lambda: argsort(X_test_feat), FORCE_REDO, SAVE_STATE)
if __DEBUG: if __DEBUG:
print("X_test_feat_argsort") print('X_test_feat_argsort')
print(X_test_feat_argsort.shape) print(X_test_feat_argsort.shape)
print(X_test_feat_argsort[IDX_INSPECT, : IDX_INSPECT_OFFSET]) print(X_test_feat_argsort[IDX_INSPECT, : IDX_INSPECT_OFFSET])
benchmark_function("Arg unit test", lambda: unit_test_argsort_2d(X_test_feat, X_test_feat_argsort)) benchmark_function('Arg unit test', lambda: unit_test_argsort_2d(X_test_feat, X_test_feat_argsort))
time_spent = perf_counter_ns() - preproc_timestamp time_spent = perf_counter_ns() - preproc_timestamp
formatted_line(preproc_gaps, '├', '┼', '─', '┤') formatted_line(preproc_gaps, '├', '┼', '─', '┤')
formatted_row(preproc_gaps, ['Preprocessing summary', f'{time_spent:,}', format_time_ns(time_spent)]) formatted_row(preproc_gaps, ['Preprocessing summary', f'{time_spent:,}', format_time_ns(time_spent)])
@ -138,16 +147,17 @@ def train(X_train_feat: np.ndarray, X_train_feat_argsort: np.ndarray, y_train: n
"""Train the weak classifiers. """Train the weak classifiers.
Args: Args:
X_train (np.ndarray): Training images. X_train (np.ndarray): Training images
X_train_feat_argsort (np.ndarray): Sorted indexes of the training images features. X_train_feat_argsort (np.ndarray): Sorted indexes of the training images features
y_train (np.ndarray): Training labels. y_train (np.ndarray): Training labels
Returns: List of trained models Returns:
List[np.ndarray]: List of trained models
""" """
training_timestamp = perf_counter_ns() training_timestamp = perf_counter_ns()
training_gaps = [26, -18, 29] training_gaps = [26, -18, 29]
header(['Training', 'Time spent (ns)', 'Formatted time spent'], training_gaps) header(training_gaps, ['Training', 'Time spent (ns)', 'Formatted time spent'])
models = [] models = []
for T in TS: for T in TS:
@ -157,9 +167,9 @@ def train(X_train_feat: np.ndarray, X_train_feat_argsort: np.ndarray, y_train: n
models.append([alphas, final_classifiers]) models.append([alphas, final_classifiers])
if __DEBUG: if __DEBUG:
print("alphas") print('alphas')
print(alphas) print(alphas)
print("final_classifiers") print('final_classifiers')
print(final_classifiers) print(final_classifiers)
time_spent = perf_counter_ns() - training_timestamp time_spent = perf_counter_ns() - training_timestamp
@ -173,15 +183,15 @@ def testing_and_evaluating(models: List[np.ndarray], X_train_feat: np.ndarray, y
"""Benchmark the trained classifiers on the training and testing sets. """Benchmark the trained classifiers on the training and testing sets.
Args: Args:
models (List[np.ndarray]): List of trained models. models (List[np.ndarray]): List of trained models
X_train_feat (np.ndarray): Training features. X_train_feat (np.ndarray): Training features
y_train (np.ndarray): Training labels. y_train (np.ndarray): Training labels
X_test_feat (np.ndarray): Testing features. X_test_feat (np.ndarray): Testing features
y_test (np.ndarray): Testing labels. y_test (np.ndarray): Testing labels
""" """
testing_gaps = [26, -19, 24, -19, 24] testing_gaps = [26, -19, 24, -19, 24]
header(['Testing', 'Time spent (ns) (E)', 'Formatted time spent (E)', 'Time spent (ns) (T)', 'Formatted time spent (T)'], testing_gaps) header(testing_gaps, ['Testing', 'Time spent (ns) (E)', 'Formatted time spent (E)', 'Time spent (ns) (T)', 'Formatted time spent (T)'])
performances = [] performances = []
total_train_timestamp = 0 total_train_timestamp = 0
@ -213,7 +223,7 @@ def testing_and_evaluating(models: List[np.ndarray], X_train_feat: np.ndarray, y
footer(testing_gaps) footer(testing_gaps)
evaluating_gaps = [19, 7, 6, 6, 6, 7, 6, 6, 6] evaluating_gaps = [19, 7, 6, 6, 6, 7, 6, 6, 6]
header(['Evaluating', 'ACC (E)', 'F1 (E)', 'FN (E)', 'FP (E)', 'ACC (T)', 'F1 (T)', 'FN (T)', 'FP (T)'], evaluating_gaps) header(evaluating_gaps, ['Evaluating', 'ACC (E)', 'F1 (E)', 'FN (E)', 'FP (E)', 'ACC (T)', 'F1 (T)', 'FN (T)', 'FP (T)'])
for T, (e_acc, e_f1, e_FN, e_FP, t_acc, t_f1, t_FN, t_FP) in zip(TS, performances): for T, (e_acc, e_f1, e_FN, e_FP, t_acc, t_f1, t_FN, t_FP) in zip(TS, performances):
print(f'│ ViolaJones T = {T:<4} │ {e_acc:>7.2%} │ {e_f1:>6.2f} │ {e_FN:>6,} │ {e_FP:>6,}', end = ' │ ') print(f'│ ViolaJones T = {T:<4} │ {e_acc:>7.2%} │ {e_f1:>6.2f} │ {e_FN:>6,} │ {e_FP:>6,}', end = ' │ ')
@ -224,7 +234,7 @@ def testing_and_evaluating(models: List[np.ndarray], X_train_feat: np.ndarray, y
def main() -> None: def main() -> None:
unit_timestamp = perf_counter_ns() unit_timestamp = perf_counter_ns()
unit_gaps = [27, -18, 29] unit_gaps = [27, -18, 29]
header(['Unit testing', 'Time spent (ns)', 'Formatted time spent'], unit_gaps) header(unit_gaps, ['Unit testing', 'Time spent (ns)', 'Formatted time spent'])
benchmark_function('testing format_time', unit_gaps[0], format_time_test) benchmark_function('testing format_time', unit_gaps[0], format_time_test)
benchmark_function('testing format_time_ns', unit_gaps[0], format_time_ns_test) benchmark_function('testing format_time_ns', unit_gaps[0], format_time_ns_test)
time_spent = perf_counter_ns() - unit_timestamp time_spent = perf_counter_ns() - unit_timestamp
@ -235,12 +245,12 @@ def main() -> None:
X_train_feat, X_train_feat_argsort, y_train, X_test_feat, y_test = preprocessing() X_train_feat, X_train_feat_argsort, y_train, X_test_feat, y_test = preprocessing()
models = train(X_train_feat, X_train_feat_argsort, y_train) models = train(X_train_feat, X_train_feat_argsort, y_train)
# X_train_feat, X_test_feat = picke_multi_loader([f"X_train_feat_{label}", f"X_test_feat_{label}"], OUT_DIR) # X_train_feat, X_test_feat = pickle_multi_loader([f'X_train_feat_{label}', f'X_test_feat_{label}'], OUT_DIR)
# indices = picke_multi_loader(["indices"], OUT_DIR)[0] # indices = pickle_multi_loader(['indices'], OUT_DIR)[0]
# X_train_feat, X_test_feat = X_train_feat[indices], X_test_feat[indices] # X_train_feat, X_test_feat = X_train_feat[indices], X_test_feat[indices]
testing_and_evaluating(models, X_train_feat, y_train, X_test_feat, y_test) testing_and_evaluating(models, X_train_feat, y_train, X_test_feat, y_test)
unit_test(TS) unit_test(TS)
if __name__ == "__main__": if __name__ == '__main__':
main() main()

189
python/test.py
View File

@ -1,189 +0,0 @@
import numpy as np
from numba import cuda, config, njit
config.CUDA_LOW_OCCUPANCY_WARNINGS = 0
#import matplotlib.pyplot as plt
from tqdm import tqdm
from time import perf_counter_ns
from toolbox import format_time_ns
from pickle import load, dump
from sys import argv
def get(a):
with open(f"{a}.pkl", 'rb') as f:
return load(f)
def save(a, name) -> None:
with open(name, 'wb') as f:
dump(a, f)
def diff(folder, a, label1, label2):
af, bf = get(f"{folder}/{a}_{label1}"), get(f"{folder}/{a}_{label2}")
#print(af)
#print(bf)
print((af - bf).mean())
if __name__ == "__main__":
if len(argv) == 5:
diff(argv[1], argv[4], argv[2], argv[3])
def py_mean(a, b):
s = 0.0
for a_i, b_i in zip(a, b):
s += a_i * b_i
return s / a.shape[0]
def np_mean(a, b):
return np.mean(a * b)
@njit('float64(float64[:], float64[:])', fastmath = True, nogil = True)
def nb_mean(a, b):
return np.mean(a * b)
@njit('float64(float64[:], float64[:])', fastmath = True, nogil = True)
def nb_mean_loop(a, b):
s = 0.0
for a_i, b_i in zip(a, b):
s += a_i * b_i
return s / a.shape[0]
@cuda.jit('void(float64[:], float64[:], float64[:])', fastmath = True)
def cuda_mean_kernel(r, a, b):
s = 0.0
for a_i, b_i in zip(a, b):
s += a_i * b_i
r[0] = s / a.shape[0]
def cuda_mean(a, b):
r = cuda.to_device(np.empty(1, dtype = np.float64))
d_a = cuda.to_device(a)
d_b = cuda.to_device(b)
cuda_mean_kernel[1, 1](r, d_a, d_b)
return r.copy_to_host()[0]
def test_and_compare(labels, fncs, a, b):
m = []
for fnc in tqdm(fncs, leave = False, desc = "Calculating..."):
s = perf_counter_ns()
m.append([fnc(a, b), perf_counter_ns() - s])
print("Results:")
[print(f"\t{label:<10} {m_i:<20} {format_time_ns(time_i)}") for ((m_i, time_i), label) in zip(m, labels)]
print("Comparaison:")
for i, (m_i, label_i) in enumerate(zip(m, labels)):
for j, (m_j, label_j) in enumerate(zip(m, labels)):
if i >= j:
continue
print(f"\t{label_i:<10} vs {label_j:<10} - {abs(m_i[0] - m_j[0])}")
if __name__ == "__main__":
np.set_printoptions(linewidth = 10000, threshold = 1000)
N = int(2**20)
labels = ["Python", "Numpy", "Numba", "Numba loop", "CUDA"]
fncs = [py_mean, np_mean, nb_mean, nb_mean_loop, cuda_mean]
print(f"RANDOM for N={N}")
total_size = (2 * 8 * N)
print(f"Size = {total_size} B")
print(f"Size = {total_size // 1024} kB")
print(f"Size = {total_size // 1024 // 1024} MB")
print(f"Size = {total_size // 1024 // 1024 // 1024} GB")
a, b = np.random.rand(N).astype(np.float64), np.random.rand(N).astype(np.float64)
test_and_compare(labels, fncs, a, b)
del a, b
print(f"\nDETERMINSTIC for N={N}")
total_size = (2 * 8 * N) + (8 * N)
print(f"Size = {total_size} B")
print(f"Size = {total_size // 1024} kB")
print(f"Size = {total_size // 1024 // 1024} MB")
print(f"Size = {total_size // 1024 // 1024 // 1024} GB")
mask = np.arange(N, dtype = np.uint64)
a = np.ones(N, dtype = np.float64)
a[mask < N//2] = 0.1
del mask
b = np.ones(N, dtype = np.float64)
test_and_compare(labels, fncs, a, b)
del a, b
#from ViolaJonesGPU import argsort as argsort_GPU
#from ViolaJonesCPU import argsort as argsort_CPU
#from toolbox import unit_test_argsort_2d, benchmark_function
#labels = ["Numpy", "Numba", "CUDA"]
#a = np.random.randint(2**12, size = (2**20, 2**8), dtype = np.int32)
#m = [benchmark_function(f"Argsort {label}", lambda: f(np.copy(a))) for (label, f) in zip(labels, [
# lambda a: np.argsort(a).astype(np.uint16), argsort_CPU, argsort_GPU
#])]
#for i, (m_i, label_i) in enumerate(zip(m, labels)):
# #for j, (m_j, label_j) in enumerate(zip(m, labels)):
# # if i >= j:
# # continue
# # print(f"\t{label_i:<10} vs {label_j:<10} - {(m_i == m_j).mean()}")
# benchmark_function(f"Unit test {label_i}", lambda: unit_test_argsort_2d(a, m_i))
#for i in tqdm(range(X.shape[0]), leave = False, desc = "Extract image"):
# x = X[i]
# y = Y[i]
# fig = plt.figure()
# plt.imshow(x, cmap = 'gray')
# plt.savefig(f"imgs/{y}/{i}.png")
# plt.close(fig)
#def extract_FD(Xy):
# X_c, Y_c = [], []
# for x,y in Xy:
# X_c.append(x)
# Y_c.append(y)
# X_c = np.asarray(X_c)
# Y_c = np.asarray(Y_c)
# return X_c, Y_c
#X_train, y_train = get('out/X_train'), get('out/y_train')
#X_test, y_test = get('out/X_test'), get('out/y_test')
#X_train, y_train = extract_FD(get('/home/_aspil0w/git/FaceDetection/training'))
#X_test, y_test = extract_FD(get('/home/_aspil0w/git/FaceDetection/test'))
#save(X_train, 'out/X_train'), save(y_train, 'out/y_train')
#save(X_test, 'out/X_test'), save(y_test, 'out/y_test')
#print(X_train.shape, X_train_org.shape, X_train.shape == X_train_org.shape)
#print((X_train == X_train_org).mean())
#print(y_train.shape, y_train_org.shape, y_train.shape == y_train_org.shape)
#print((y_train == y_train_org).mean())
#print(X_test.shape, X_test_org.shape, X_test.shape == X_test_org.shape)
#print((X_test == X_test_org).mean())
#print(y_test.shape, y_test_org.shape, y_test.shape == y_test_org.shape)
#print((y_test == y_test_org).mean())
#@njit('uint16[:](uint8[:, :, :], uint8[:, :, :])')
#def arg_find(X, X_org):
# arg = np.empty(X.shape[0], dtype = np.uint16)
# for i, x in enumerate(X_org):
# found = False
# for j, x_org in enumerate(X):
# if np.all(x == x_org):
# arg[i] = j
# found = True
# break
# assert found, "Image not found"
# return arg
#print("Arg find results train")
#arg_train = arg_find(X_train, X_train_org)
#print((X_train[arg_train] == X_train_org).mean())
#print((y_train[arg_train] == y_train_org).mean())
#print("Arg find results test")
#arg_test = arg_find(X_test, X_test_org)
#print((X_test[arg_test] == X_test_org).mean())
#print((y_test[arg_test] == y_test_org).mean())
#for i in tqdm(range(X_c.shape[0]), leave = False, desc = "Extract image"):
# x = X_c[i]
# y = Y_c[i]
# fig = plt.figure()
# plt.imshow(x, cmap = 'gray')
# plt.savefig(f"imgs2/{y}/{i}.png")
# plt.close(fig)

130
python/toolbox.py
View File

@ -1,6 +1,5 @@
from typing import Any, Callable, List, Union, Final from typing import Any, Callable, List, Union, Final
from time import perf_counter_ns from time import perf_counter_ns
from numba import njit
import numpy as np import numpy as np
from sys import stderr from sys import stderr
import pickle import pickle
@ -8,70 +7,122 @@ import os
from config import MODEL_DIR, OUT_DIR from config import MODEL_DIR, OUT_DIR
from decorators import njit from decorators import njit
time_formats: Final = ["ns", "µs", "ms", "s", "m", "h", "j", "w", "M", "y", "c"]
def formatted_row(gaps: list[int], titles: list[str], separator: str = '│') -> None: def formatted_row(gaps: list[int], titles: list[str], separator: str = '│') -> None:
"""Print a formatted row of titles with of gaps seperated by a separator.
Args:
gaps: List of size gaps
titles: List of titles
separator: Separator character between each gap
"""
for gap, title in zip(gaps, titles): for gap, title in zip(gaps, titles):
print(f"{separator} {title:{'>' if gap < 0 else '<'}{abs(gap)}} ", end = '') print(f"{separator} {title:{'>' if gap < 0 else '<'}{abs(gap)}} ", end = '')
print(separator) print(separator)
def formatted_line(gaps: list[int], right: str, middle: str, separator: str, left: str) -> None: def formatted_line(gaps: list[int], left: str, middle: str, separator: str, right: str) -> None:
print(right, end = '') """Print a formatted line of repeated characters.
Args:
gaps: List of size gaps
left: Character on the left
middle: Character between each separator
separator: Separator character between each gap
right: Character on the right
"""
print(left, end = '')
last_gap = len(gaps) - 1 last_gap = len(gaps) - 1
for i, gap in enumerate(gaps): for i, gap in enumerate(gaps):
print(f'{separator * (abs(gap) + 2)}', end = '') print(f'{separator * (abs(gap) + 2)}', end = '')
if i != last_gap: if i != last_gap:
print(middle, end = '') print(middle, end = '')
print(left) print(right)
def header(titles: list[str], gaps: list[int]) -> None: def header(gaps: list[int], titles: list[str]) -> None:
"""Print a formatted header with the given titles and sizes.
Args:
gaps: List of size gaps
titles: List of titles
"""
formatted_line(gaps, '┌', '┬', '─', '┐') formatted_line(gaps, '┌', '┬', '─', '┐')
formatted_row(gaps, titles) formatted_row(gaps, titles)
formatted_line(gaps, '├', '┼', '─', '┤') formatted_line(gaps, '├', '┼', '─', '┤')
def footer(gaps: list[int]) -> None: def footer(gaps: list[int]) -> None:
"""Print a formatted fooder with the given sizes
Args:
gaps: List of size gaps
"""
formatted_line(gaps, '└', '┴', '─', '┘') formatted_line(gaps, '└', '┴', '─', '┘')
time_formats: Final = ['ns', 'µs', 'ms', 's', 'm', 'h', 'j', 'w', 'M', 'y', 'c']
time_numbers: Final = np.array([1, 1e3, 1e6, 1e9, 6e10, 36e11, 864e11, 6048e11, 26784e11, 31536e12, 31536e14], dtype = np.uint64) time_numbers: Final = np.array([1, 1e3, 1e6, 1e9, 6e10, 36e11, 864e11, 6048e11, 26784e11, 31536e12, 31536e14], dtype = np.uint64)
@njit('str(uint64)') @njit('str(uint64)')
def format_time_ns(time: int) -> str: def format_time_ns(time: int) -> str:
"""Format the time in nanoseconds in human readable format. """Format the time in nanoseconds in human readable format.
Args: Args:
time (int): Time in nanoseconds. time (int): Time in nanoseconds
Returns: Returns:
str: The formatted human readable string. str: The formatted human readable string
""" """
assert time >= 0, "Incorrect time stamp" assert time >= 0, 'Incorrect time stamp'
if time == 0: if time == 0:
return "0ns" return '0ns'
s = "" s = ''
for i in range(time_numbers.shape[0])[::-1]: for i in range(time_numbers.shape[0])[::-1]:
if time >= time_numbers[i]: if time >= time_numbers[i]:
res = int(time // time_numbers[i]) res = int(time // time_numbers[i])
time = time % time_numbers[i] time = time % time_numbers[i]
s += f"{res}{time_formats[i]} " s += f'{res}{time_formats[i]} '
assert time == 0, "Leftover in formatting time !" assert time == 0, 'Leftover in formatting time !'
return s.rstrip() return s.rstrip()
def picke_multi_loader(filenames: List[str], save_dir: str = MODEL_DIR) -> List[Any]: @njit('str(uint64)')
def format_time(time: int) -> str:
"""Format the time in seconds in human readable format.
Args:
time (int): Time in seconds
Returns:
str: The formatted human readable string
"""
assert time >= 0, 'Incorrect time stamp'
if time == 0:
return '0s'
s = ''
for i in range(3, time_numbers.shape[0])[::-1]:
time_number = time_numbers[i] / int(1e9)
if time >= time_number:
res = int(time // time_number)
time = time % time_number
s += f'{res}{time_formats[i]} '
assert time == 0, 'Leftover in formatting time !'
return s.rstrip()
def pickle_multi_loader(filenames: List[str], save_dir: str = MODEL_DIR) -> List[Any]:
"""Load multiple pickle data files. """Load multiple pickle data files.
Args: Args:
filenames (List[str]): List of all the filename to load. filenames (List[str]): List of all the filename to load
save_dir (str, optional): Path of the files to load. Defaults to MODELS_DIR (see config.py). save_dir (str, optional): Path of the files to load. Defaults to MODELS_DIR (see config.py)
Returns: Returns:
List[Any]. List of loaded pickle data files. List[Any]. List of loaded pickle data files
""" """
b = [] b = []
for f in filenames: for f in filenames:
filepath = f"{save_dir}/{f}.pkl" filepath = f'{save_dir}/{f}.pkl'
if os.path.exists(filepath): if os.path.exists(filepath):
with open(filepath, "rb") as filebyte: with open(filepath, 'rb') as file_bytes:
b.append(pickle.load(filebyte)) b.append(pickle.load(file_bytes))
else: else:
b.append(None) b.append(None)
return b return b
@ -80,11 +131,11 @@ def benchmark_function(step_name: str, column_width: int, fnc: Callable) -> Any:
"""Benchmark a function and display the result of stdout. """Benchmark a function and display the result of stdout.
Args: Args:
step_name (str): Name of the function to call. step_name (str): Name of the function to call
fnc (Callable): Function to call. fnc (Callable): Function to call
Returns: Returns:
Any: Result of the function. Any: Result of the function
""" """
print(f'{step_name}...', file = stderr, end = '\r') print(f'{step_name}...', file = stderr, end = '\r')
s = perf_counter_ns() s = perf_counter_ns()
@ -98,34 +149,34 @@ def state_saver(step_name: str, column_width: int, filename: Union[str, List[str
"""Either execute a function then saves the result or load the already existing result. """Either execute a function then saves the result or load the already existing result.
Args: Args:
step_name (str): Name of the function to call. step_name (str): Name of the function to call
filename (Union[str, List[str]]): Name or list of names of the filenames where the result(s) are saved. filename (Union[str, List[str]]): Name or list of names of the filenames where the result(s) are saved
fnc ([type]): Function to call. fnc ([type]): Function to call
force_redo (bool, optional): Recall the function even if the result(s) is already saved. Defaults to False. force_redo (bool, optional): Recall the function even if the result(s) is already saved. Defaults to False
save_dir (str, optional): Path of the directory to save the result(s). Defaults to OUT_DIR (see config.py). save_dir (str, optional): Path of the directory to save the result(s). Defaults to OUT_DIR (see config.py)
Returns: Returns:
Any: The result(s) of the called function Any: The result(s) of the called function
""" """
if isinstance(filename, str): if isinstance(filename, str):
if not os.path.exists(f"{save_dir}/{filename}.pkl") or force_redo: if not os.path.exists(f'{save_dir}/{filename}.pkl') or force_redo:
b = benchmark_function(step_name, column_width, fnc) b = benchmark_function(step_name, column_width, fnc)
if save_state: if save_state:
with open(f"{save_dir}/{filename}.pkl", 'wb') as f:
print(f'Saving results of {step_name}', file = stderr, end = '\r') print(f'Saving results of {step_name}', file = stderr, end = '\r')
with open(f'{save_dir}/{filename}.pkl', 'wb') as f:
pickle.dump(b, f) pickle.dump(b, f)
print(' ' * 100, file = stderr, end = '\r') print(' ' * 100, file = stderr, end = '\r')
return b return b
else: else:
with open(f"{save_dir}/{filename}.pkl", "rb") as f:
print(f'Loading results of {step_name}', file = stderr, end = '\r') print(f'Loading results of {step_name}', file = stderr, end = '\r')
with open(f'{save_dir}/{filename}.pkl', 'rb') as f:
res = pickle.load(f) res = pickle.load(f)
print(f"│ {step_name:<{column_width}} │ {'None':>18} │ {'loaded saved state':<29} │") print(f"│ {step_name:<{column_width}} │ {'None':>18} │ {'loaded saved state':<29} │")
return res return res
elif isinstance(filename, list): elif isinstance(filename, list):
abs = False abs = False
for fn in filename: for fn in filename:
if not os.path.exists(f"{save_dir}/{fn}.pkl"): if not os.path.exists(f'{save_dir}/{fn}.pkl'):
abs = True abs = True
break break
if abs or force_redo: if abs or force_redo:
@ -133,7 +184,7 @@ def state_saver(step_name: str, column_width: int, filename: Union[str, List[str
if save_state: if save_state:
print(f'Saving results of {step_name}', file = stderr, end = '\r') print(f'Saving results of {step_name}', file = stderr, end = '\r')
for bi, fnI in zip(b, filename): for bi, fnI in zip(b, filename):
with open(f"{save_dir}/{fnI}.pkl", 'wb') as f: with open(f'{save_dir}/{fnI}.pkl', 'wb') as f:
pickle.dump(bi, f) pickle.dump(bi, f)
print(' ' * 100, file = stderr, end = '\r') print(' ' * 100, file = stderr, end = '\r')
return b return b
@ -142,15 +193,24 @@ def state_saver(step_name: str, column_width: int, filename: Union[str, List[str
b = [] b = []
print(f'Loading results of {step_name}', file = stderr, end = '\r') print(f'Loading results of {step_name}', file = stderr, end = '\r')
for fn in filename: for fn in filename:
with open(f"{save_dir}/{fn}.pkl", "rb") as f: with open(f'{save_dir}/{fn}.pkl', 'rb') as f:
b.append(pickle.load(f)) b.append(pickle.load(f))
print(' ' * 100, file = stderr, end = '\r') print(' ' * 100, file = stderr, end = '\r')
return b return b
else: else:
assert False, f"Incompatible filename type = {type(filename)}" assert False, f'Incompatible filename type = {type(filename)}'
@njit('boolean(int32[:, :], uint16[:, :])') @njit('boolean(int32[:, :], uint16[:, :])')
def unit_test_argsort_2d(arr: np.ndarray, indices: np.ndarray) -> bool: def unit_test_argsort_2d(arr: np.ndarray, indices: np.ndarray) -> bool:
"""Test if a given array of indices sort a given array.
Args:
arr (np.ndarray): Array of data
indices (np.ndarray): Indices that sort arr
Returns:
bool: Success of the test
"""
n = indices.shape[0] n = indices.shape[0]
total = indices.shape[0] * indices.shape[1] total = indices.shape[0] * indices.shape[1]
for i, sub_indices in enumerate(indices): for i, sub_indices in enumerate(indices):

183
python/toolbox_unit_test.py
View File

@ -1,67 +1,132 @@
from typing import Any from typing import Any
from toolbox import format_time_ns from toolbox import format_time, format_time_ns
def Assert(name: str, expected: Any, result: Any): def Assert(name: str, expected: Any, result: Any):
"""Test if a given result is equal of the expected one and log result
Args:
name (str): name of the unit test
expected (Any): expected result of the function call
result (Any): result of the function
"""
if expected != result: if expected != result:
print(f"For test name {name} Expected '{expected}' but got '{result}' instead") print(f"For test name {name} Expected '{expected}' but got '{result}' instead")
assert False assert False
def format_time_ns_test() -> None: def format_time_test() -> None:
# https://en.wikipedia.org/wiki/Unit_of_time """Test suite for the format_time output
Assert("format_time_ns null", "0ns", format_time_ns(0));
Assert("format_time_ns nanosecond", "1ns", format_time_ns(1));
Assert("format_time_ns shake", "10ns", format_time_ns(10));
Assert("format_time_ns microsecond", "1µs", format_time_ns(int(1e3)));
Assert("format_time_ns millisecond", "1ms", format_time_ns(int(1e6)));
Assert("format_time_ns centisecond", "10ms", format_time_ns(int(1e7)));
Assert("format_time_ns decisecond", "100ms", format_time_ns(int(1e8)));
Assert("format_time_ns second", "1s", format_time_ns(int(1e9)));
Assert("format_time_ns decasecond", "10s", format_time_ns(int(1e10)));
Assert("format_time_ns minute", "1m", format_time_ns(int(6e10)));
Assert("format_time_ns milliday", "1m 26s 400ms", format_time_ns(int(864e8)));
Assert("format_time_ns hectosecond", "1m 40s", format_time_ns(int(1e11)));
Assert("format_time_ns kilosecond", "16m 40s", format_time_ns(int(1e12)));
Assert("format_time_ns hour", "1h", format_time_ns(int(36e11)));
Assert("format_time_ns day", "1j", format_time_ns(int(864e11)));
Assert("format_time_ns week/sennight", "1w", format_time_ns(int(6048e11)));
Assert("format_time_ns megasecond", "1w 4j 13h 46m 40s", format_time_ns(int(1e15)));
Assert("format_time_ns fortnight", "2w", format_time_ns(int(12096e11)));
Assert("format_time_ns lunar month (draconitic)", "3w 6j 5h 5m 35s 800ms", format_time_ns(int(23511358e8)));
Assert("format_time_ns lunar month (tropical)", "3w 6j 7h 43m 4s 700ms", format_time_ns(int(23605847e8)));
Assert("format_time_ns lunar month (sidereal)", "3w 6j 7h 43m 11s 600ms", format_time_ns(int(23605916e8)));
Assert("format_time_ns lunar month (anomalistic)", "3w 6j 13h 18m 33s 200ms", format_time_ns(int(23807132e8)));
Assert("format_time_ns lunar month (synodic)", "4w 1j 12h 44m 2s 900ms", format_time_ns(int(25514429e8)));
Assert("format_time_ns month", "1M", format_time_ns(int(26784e11)));
Assert("format_time_ns quarantine", "1M 1w 2j", format_time_ns(int(3456e12)));
Assert("format_time_ns semester", "4M 2j", format_time_ns(int(108864e11)));
Assert("format_time_ns lunar year", "11M 1w 6j 8h 52m 48s", format_time_ns(int(30617568e9)));
Assert("format_time_ns year", "1y", format_time_ns(int(31536e12)));
Assert("format_time_ns tropical year", "1y 5h 48m 45s 216ms", format_time_ns(int(31556925216e6)));
Assert("format_time_ns gregorian year", "1y 5h 49m 12s", format_time_ns(int(31556952e9)));
Assert("format_time_ns sidereal year", "1y 6h 9m 9s 763ms 545µs 600ns", format_time_ns(int(315581497635456e2)));
Assert("format_time_ns leap year", "1y 1j", format_time_ns(int(316224e11)));
Assert("format_time_ns olympiad", "4y", format_time_ns(int(126144e12)));
Assert("format_time_ns lusturm", "5y", format_time_ns(int(15768e13)));
Assert("format_time_ns decade", "10y", format_time_ns(int(31536e13)));
Assert("format_time_ns indiction", "15y", format_time_ns(int(47304e13)));
Assert("format_time_ns score", "20y", format_time_ns(int(63072e13)));
Assert("format_time_ns gigasecond", "31y 8M 1w 4j 1h 46m 40s", format_time_ns(int(1e18)));
Assert("format_time_ns jubilee", "50y", format_time_ns(int(15768e14)));
Assert("format_time_ns century", "1c", format_time_ns(int(31536e14)));
Assert("format_time_ns millennium", "10c", format_time_ns(int(31536e15)));
Assert("format_time_ns age", "257c 72y", format_time_ns(int(812745792e12)));
Assert("format_time_ns terasecond", "3170c 97y 10M 3w 4j 17h 46m 40s", format_time_ns(int(1e22)));
Assert("format_time_ns megaannum", "10000c", format_time_ns(int(31536e18)));
# Cannot use number bigger than currently supported ISO Python
#Assert("format_time_ns petasecond", "317097c 91y 11M 2w 4j 1h 46m 40s", format_time_ns(int(1e24)));
#Assert("format_time_ns galactic year", "2300000c", format_time_ns(int(725328e19)));
#Assert("format_time_ns eon", "10000000c", format_time_ns(int(31536e21)));
#Assert("format_time_ns kalpa", "43200000c", format_time_ns(int(13623552e19)));
#Assert("format_time_ns exasecond", "317097919c 83y 9M 1h 46m 40s", format_time_ns(int(1e27)));
#Assert("format_time_ns zettasecond", "", format_time_ns(int(1e30)));
#Assert("format_time_ns yottasecond", "", format_time_ns(int(1e33)));
#Assert("format_time_ns ronnasecond", "", format_time_ns(int(1e36)));
#Assert("format_time_ns quettasecond", "", format_time_ns(int(1e39)));
# uint64_t_MAX == 2**64 == 18446744073709551615I64u == -1
Assert("format_time_ns max", "5c 84y 11M 2j 23h 34m 33s 709ms 551µs 615ns", format_time_ns(2**64 - 1))
See https://en.wikipedia.org/wiki/Unit_of_time for details
"""
Assert("format_time null", "0s", format_time(0))
Assert("format_time second", "1s", format_time(1))
Assert("format_time decasecond", "10s", format_time(10))
Assert("format_time minute", "1m", format_time(60))
Assert("format_time milliday", "1m 26s", format_time(86)) # missing 0.4s due to precision
Assert("format_time hectosecond", "1m 40s", format_time(100))
Assert("format_time kilosecond", "16m 40s", format_time(int(1e3)))
Assert("format_time hour", "1h", format_time(3600))
Assert("format_time day", "1j", format_time(86400))
Assert("format_time week/sennight", "1w", format_time(604800))
Assert("format_time megasecond", "1w 4j 13h 46m 40s", format_time(int(1e6)))
Assert("format_time fortnight", "2w", format_time(1209600))
Assert("format_time lunar month (draconitic)", "3w 6j 5h 5m 35s", format_time(2351135)) # missing 0.8 due to precision
Assert("format_time lunar month (tropical)", "3w 6j 7h 43m 4s", format_time(2360584)) # missing 0.7 due to precision
Assert("format_time lunar month (sidereal)", "3w 6j 7h 43m 11s", format_time(2360591)) # missing 0.6 to precision
Assert("format_time lunar month (anomalistic)", "3w 6j 13h 18m 33s", format_time(2380713)) # missing 0.2 due to precision
Assert("format_time lunar month (synodic)", "4w 1j 12h 44m 2s", format_time(2551442)) # missing 0.9 due to precision
Assert("format_time month", "1M", format_time(2678400))
Assert("format_time quarantine", "1M 1w 2j", format_time(int(3456e3)))
Assert("format_time semester", "4M 2j", format_time(10886400))
Assert("format_time lunar year", "11M 1w 6j 8h 52m 48s", format_time(30617568))
Assert("format_time year", "1y", format_time(int(31536e3)))
Assert("format_time tropical year", "1y 5h 48m 45s", format_time(31556925)) # missing 0.216 due to precision
Assert("format_time gregorian year", "1y 5h 49m 12s", format_time(31556952))
Assert("format_time sidereal year", "1y 6h 9m 9s", format_time(31558149)) # missing 0.7635456 due to precision
Assert("format_time leap year", "1y 1j", format_time(31622400))
Assert("format_time olympiad", "4y", format_time(int(126144e3)))
Assert("format_time lusturm", "5y", format_time(int(15768e4)))
Assert("format_time decade", "10y", format_time(int(31536e4)))
Assert("format_time indiction", "15y", format_time(int(47304e4)))
Assert("format_time score", "20y", format_time(int(63072e4)))
Assert("format_time gigasecond", "31y 8M 1w 4j 1h 46m 40s", format_time(int(1e9)))
Assert("format_time jubilee", "50y", format_time(int(15768e5)))
Assert("format_time century", "1c", format_time(int(31536e5)))
Assert("format_time millennium", "10c", format_time(int(31536e6)))
Assert("format_time age", "257c 72y", format_time(int(812745792e3)))
Assert("format_time terasecond", "3170c 97y 10M 3w 4j 17h 46m 40s", format_time(int(1e13)))
Assert("format_time megaannum", "10000c", format_time(int(31536e9)))
Assert("format_time petasecond", "317097c 91y 11M 2w 4j 1h 46m 40s", format_time(int(1e15)))
Assert("format_time galactic year", "2300000c", format_time(int(725328e10)))
Assert("format_time eon", "10000000c", format_time(int(31536e12)))
Assert("format_time kalpa", "43200000c", format_time(int(13623552e10)))
Assert("format_time exasecond", "317097919c 83y 9M 1h 46m 40s", format_time(int(1e18)))
# Cannot use number bigger than currently supported ISO Python
#Assert("format_time zettasecond", "", format_time(1e21))
#Assert("format_time yottasecond", "", format_time(1e24))
#Assert("format_time ronnasecond", "", format_time(1e27))
#Assert("format_time quettasecond", "", format_time(1e30))
# uint64_t_MAX == 2**64 == 18446744073709551615 == -1
Assert("format_time max", "5849424173c 55y 3w 5j 7h 16s", format_time(int(2**64 - 1)))
def format_time_ns_test() -> None:
"""Test suite for the format_time_ns output
See https://en.wikipedia.org/wiki/Unit_of_time for details
"""
Assert("format_time_ns null", "0ns", format_time_ns(0))
Assert("format_time_ns nanosecond", "1ns", format_time_ns(1))
Assert("format_time_ns shake", "10ns", format_time_ns(10))
Assert("format_time_ns microsecond", "1µs", format_time_ns(int(1e3)))
Assert("format_time_ns millisecond", "1ms", format_time_ns(int(1e6)))
Assert("format_time_ns centisecond", "10ms", format_time_ns(int(1e7)))
Assert("format_time_ns decisecond", "100ms", format_time_ns(int(1e8)))
Assert("format_time_ns second", "1s", format_time_ns(int(1e9)))
Assert("format_time_ns decasecond", "10s", format_time_ns(int(1e10)))
Assert("format_time_ns minute", "1m", format_time_ns(int(6e10)))
Assert("format_time_ns milliday", "1m 26s 400ms", format_time_ns(int(864e8)))
Assert("format_time_ns hectosecond", "1m 40s", format_time_ns(int(1e11)))
Assert("format_time_ns kilosecond", "16m 40s", format_time_ns(int(1e12)))
Assert("format_time_ns hour", "1h", format_time_ns(int(36e11)))
Assert("format_time_ns day", "1j", format_time_ns(int(864e11)))
Assert("format_time_ns week/sennight", "1w", format_time_ns(int(6048e11)))
Assert("format_time_ns megasecond", "1w 4j 13h 46m 40s", format_time_ns(int(1e15)))
Assert("format_time_ns fortnight", "2w", format_time_ns(int(12096e11)))
Assert("format_time_ns lunar month (draconitic)", "3w 6j 5h 5m 35s 800ms", format_time_ns(int(23511358e8)))
Assert("format_time_ns lunar month (tropical)", "3w 6j 7h 43m 4s 700ms", format_time_ns(int(23605847e8)))
Assert("format_time_ns lunar month (sidereal)", "3w 6j 7h 43m 11s 600ms", format_time_ns(int(23605916e8)))
Assert("format_time_ns lunar month (anomalistic)", "3w 6j 13h 18m 33s 200ms", format_time_ns(int(23807132e8)))
Assert("format_time_ns lunar month (synodic)", "4w 1j 12h 44m 2s 900ms", format_time_ns(int(25514429e8)))
Assert("format_time_ns month", "1M", format_time_ns(int(26784e11)))
Assert("format_time_ns quarantine", "1M 1w 2j", format_time_ns(int(3456e12)))
Assert("format_time_ns semester", "4M 2j", format_time_ns(int(108864e11)))
Assert("format_time_ns lunar year", "11M 1w 6j 8h 52m 48s", format_time_ns(int(30617568e9)))
Assert("format_time_ns year", "1y", format_time_ns(int(31536e12)))
Assert("format_time_ns tropical year", "1y 5h 48m 45s 216ms", format_time_ns(int(31556925216e6)))
Assert("format_time_ns gregorian year", "1y 5h 49m 12s", format_time_ns(int(31556952e9)))
Assert("format_time_ns sidereal year", "1y 6h 9m 9s 763ms 545µs 600ns", format_time_ns(int(315581497635456e2)))
Assert("format_time_ns leap year", "1y 1j", format_time_ns(int(316224e11)))
Assert("format_time_ns olympiad", "4y", format_time_ns(int(126144e12)))
Assert("format_time_ns lusturm", "5y", format_time_ns(int(15768e13)))
Assert("format_time_ns decade", "10y", format_time_ns(int(31536e13)))
Assert("format_time_ns indiction", "15y", format_time_ns(int(47304e13)))
Assert("format_time_ns score", "20y", format_time_ns(int(63072e13)))
Assert("format_time_ns gigasecond", "31y 8M 1w 4j 1h 46m 40s", format_time_ns(int(1e18)))
Assert("format_time_ns jubilee", "50y", format_time_ns(int(15768e14)))
Assert("format_time_ns century", "1c", format_time_ns(int(31536e14)))
Assert("format_time_ns millennium", "10c", format_time_ns(int(31536e15)))
Assert("format_time_ns age", "257c 72y", format_time_ns(int(812745792e12)))
Assert("format_time_ns terasecond", "3170c 97y 10M 3w 4j 17h 46m 40s", format_time_ns(int(1e22)))
Assert("format_time_ns megaannum", "10000c", format_time_ns(int(31536e18)))
# Cannot use number bigger than currently supported ISO Python
# Assert("format_time_ns petasecond", "317097c 91y 11M 2w 4j 1h 46m 40s", format_time_ns(int(1e24)))
# Assert("format_time_ns galactic year", "2300000c", format_time_ns(int(725328e19)))
# Assert("format_time_ns eon", "10000000c", format_time_ns(int(31536e21)))
# Assert("format_time_ns kalpa", "43200000c", format_time_ns(int(13623552e19)))
# Assert("format_time_ns exasecond", "317097919c 83y 9M 1h 46m 40s", format_time_ns(int(1e27)))
# Assert("format_time_ns zettasecond", "", format_time_ns(int(1e30)))
# Assert("format_time_ns yottasecond", "", format_time_ns(int(1e33)))
# Assert("format_time_ns ronnasecond", "", format_time_ns(int(1e36)))
# Assert("format_time_ns quettasecond", "", format_time_ns(int(1e39)))
# uint64_t_MAX == 2**64 == 18446744073709551615 == -1
Assert("format_time_ns max", "5c 84y 11M 2j 23h 34m 33s 709ms 551µs 615ns", format_time_ns(2**64 - 1))