saundersp/ViolaJones
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

python : Updated code with better display, documentation and format_time

Browse Source
This commit is contained in:
saundersp 2024-04-28 00:25:13 +02:00
parent c7d21e1014
commit 718724b63b
11 changed files with 591 additions and 566 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
python/ViolaJones.py
View File

@ -21,10 +21,10 @@ def build_features(width: int, height: int) -> np.ndarray:
"""Initialize the features base on the input shape.
Args:
shape (Tuple[int, int]): Shape of the image (Width, Height).
shape (Tuple[int, int]): Shape of the image (Width, Height)
Returns:
np.ndarray: The initialized features.
np.ndarray: The initialized features
"""
feats = []
empty = (0, 0, 0, 0)
@ -63,10 +63,10 @@ def init_weights(y_train: np.ndarray) -> np.ndarray:
"""Initialize the weights of the weak classifiers based on the training labels.
Args:
y_train (np.ndarray): Training labels.
y_train (np.ndarray): Training labels
Returns:
np.ndarray: The initialized weights.
np.ndarray: The initialized weights
"""
weights = np.empty_like(y_train, dtype = np.float64)
t = y_train.sum()
@ -79,12 +79,12 @@ def classify_weak_clf(x_feat_i: np.ndarray, threshold: int, polarity: int) -> np
"""Classify the integrated features based on polarity and threshold.
Args:
x_feat_i (np.ndarray): Integrated features.
threshold (int): Trained threshold.
polarity (int): Trained polarity.
x_feat_i (np.ndarray): Integrated features
threshold (int): Trained threshold
polarity (int): Trained polarity
Returns:
np.ndarray: Classified features.
np.ndarray: Classified features
"""
res = np.zeros_like(x_feat_i, dtype = np.int8)
res[polarity * x_feat_i < polarity * threshold] = 1
@ -95,10 +95,10 @@ def select_best(classifiers: np.ndarray, weights: np.ndarray, X_feat: np.ndarray
"""Select the best classifier given theirs predictions.
Args:
classifiers (np.ndarray): The weak classifiers.
weights (np.ndarray): Trained weights of each classifiers.
X_feat (np.ndarray): Integrated features.
y (np.ndarray): Features labels.
classifiers (np.ndarray): The weak classifiers
weights (np.ndarray): Trained weights of each classifiers
X_feat (np.ndarray): Integrated features
y (np.ndarray): Features labels
Returns:
Tuple[int, float, np.ndarray]: Index of the best classifier, the best error and the best accuracy
@ -116,13 +116,13 @@ def train_viola_jones(T: int, X_feat: np.ndarray, X_feat_argsort: np.ndarray, y:
"""Train the weak classifiers.
Args:
T (int): Number of weak classifiers.
X_feat (np.ndarray): Integrated features.
X_feat_argsort (np.ndarray): Sorted indexes of the integrated features.
y (np.ndarray): Features labels.
T (int): Number of weak classifiers
X_feat (np.ndarray): Integrated features
X_feat_argsort (np.ndarray): Sorted indexes of the integrated features
y (np.ndarray): Features labels
Returns:
Tuple[np.ndarray, np.ndarray]: List of trained alphas and the list of the final classifiers.
Tuple[np.ndarray, np.ndarray]: List of trained alphas and the list of the final classifiers
"""
weights = init_weights(y)
alphas, final_classifier = np.empty(T, dtype = np.float64), np.empty((T, 3), dtype = np.int32)
@ -144,12 +144,12 @@ def classify_viola_jones(alphas: np.ndarray, classifiers: np.ndarray, X_feat: np
"""Classify the trained classifiers on the given features.
Args:
alphas (np.ndarray): Trained alphas.
classifiers (np.ndarray): Trained classifiers.
X_feat (np.ndarray): Integrated features.
alphas (np.ndarray): Trained alphas
classifiers (np.ndarray): Trained classifiers
X_feat (np.ndarray): Integrated features
Returns:
np.ndarray: Classification results.
np.ndarray: Classification results
"""
total = np.zeros(X_feat.shape[1], dtype = np.float64)
@ -161,22 +161,22 @@ def classify_viola_jones(alphas: np.ndarray, classifiers: np.ndarray, X_feat: np
y_pred[total >= 0.5 * np.sum(alphas)] = 1
return y_pred
@njit
def get_best_anova_features(X: np.ndarray, y: np.ndarray) -> np.ndarray:
#SelectPercentile(f_classif, percentile = 10).fit(X, y).get_support(indices = True)
classes = [X.T[y == 0].astype(np.float64), X.T[y == 1].astype(np.float64)]
n_samples_per_class = np.asarray([classes[0].shape[0], classes[1].shape[0]])
n_samples = classes[0].shape[0] + classes[1].shape[0]
ss_alldata = (classes[0] ** 2).sum(axis = 0) + (classes[1] ** 2).sum(axis = 0)
sums_classes = [np.asarray(classes[0].sum(axis = 0)), np.asarray(classes[1].sum(axis = 0))]
sq_of_sums_alldata = (sums_classes[0] + sums_classes[1]) ** 2
sq_of_sums_args = [sums_classes[0] ** 2, sums_classes[1] ** 2]
ss_tot = ss_alldata - sq_of_sums_alldata / n_samples
sqd_sum_bw_n = sq_of_sums_args[0] / n_samples_per_class[0] + \
sq_of_sums_args[1] / n_samples_per_class[1] - sq_of_sums_alldata / n_samples
ss_wn = ss_tot - sqd_sum_bw_n
df_wn = n_samples - 2
msw = ss_wn / df_wn
f_values = sqd_sum_bw_n / msw
return np.sort(np.argsort(f_values)[::-1][: int(np.ceil(X.shape[0] / 10.0))])
#@njit
#def get_best_anova_features(X: np.ndarray, y: np.ndarray) -> np.ndarray:
# #SelectPercentile(f_classif, percentile = 10).fit(X, y).get_support(indices = True)
# classes = [X.T[y == 0].astype(np.float64), X.T[y == 1].astype(np.float64)]
# n_samples_per_class = np.asarray([classes[0].shape[0], classes[1].shape[0]])
# n_samples = classes[0].shape[0] + classes[1].shape[0]
# ss_all_data = (classes[0] ** 2).sum(axis = 0) + (classes[1] ** 2).sum(axis = 0)
# sums_classes = [np.asarray(classes[0].sum(axis = 0)), np.asarray(classes[1].sum(axis = 0))]
# sq_of_sums_all_data = (sums_classes[0] + sums_classes[1]) ** 2
# sq_of_sums_args = [sums_classes[0] ** 2, sums_classes[1] ** 2]
# ss_tot = ss_all_data - sq_of_sums_all_data / n_samples
#
# sqd_sum_bw_n = sq_of_sums_args[0] / n_samples_per_class[0] + \
# sq_of_sums_args[1] / n_samples_per_class[1] - sq_of_sums_all_data / n_samples
# ss_wn = ss_tot - sqd_sum_bw_n
# df_wn = n_samples - 2
# msw = ss_wn / df_wn
# f_values = sqd_sum_bw_n / msw
# return np.sort(np.argsort(f_values)[::-1][: int(np.ceil(X.shape[0] / 10.0))])

161
python/ViolaJonesCPU.py
View File

@ -18,10 +18,10 @@ def set_integral_image(X: np.ndarray) -> np.ndarray:
"""Transform the input images in integrated images (CPU version).
Args:
X (np.ndarray): Dataset of images.
X (np.ndarray): Dataset of images
Returns:
np.ndarray: Dataset of integrated images.
np.ndarray: Dataset of integrated images
"""
X_ii = np.empty_like(X, dtype = np.uint32)
for i, Xi in enumerate(tqdm_iter(X, "Applying integral image")):
@ -34,59 +34,18 @@ def set_integral_image(X: np.ndarray) -> np.ndarray:
X_ii[i] = ii
return X_ii
@njit('uint32(uint32[:, :], int16, int16, int16, int16)')
def __compute_feature__(ii: np.ndarray, x: int, y: int, w: int, h: int) -> int:
"""Compute a feature on an integrated image at a specific coordinate (CPU version).
Args:
ii (np.ndarray): Integrated image.
x (int): X coordinate.
y (int): Y coordinate.
w (int): width of the feature.
h (int): height of the feature.
Returns:
int: Computed feature.
"""
return ii[y + h, x + w] + ii[y, x] - ii[y + h, x] - ii[y, x + w]
@njit('int32[:, :](uint8[:, :, :, :], uint32[:, :, :])')
def apply_features(feats: np.ndarray, X_ii: np.ndarray) -> np.ndarray:
"""Apply the features on a integrated image dataset (CPU version).
Args:
feats (np.ndarray): Features to apply.
X_ii (np.ndarray): Integrated image dataset.
Returns:
np.ndarray: Applied features.
"""
X_feat = np.empty((feats.shape[0], X_ii.shape[0]), dtype = np.int32)
for i, (p, n) in enumerate(tqdm_iter(feats, "Applying features")):
for j, x_i in enumerate(X_ii):
p_x, p_y, p_w, p_h = p[0]
p1_x, p1_y, p1_w, p1_h = p[1]
n_x, n_y, n_w, n_h = n[0]
n1_x, n1_y, n1_w, n1_h = n[1]
p1 = __compute_feature__(x_i, p_x, p_y, p_w, p_h) + __compute_feature__(x_i, p1_x, p1_y, p1_w, p1_h)
n1 = __compute_feature__(x_i, n_x, n_y, n_w, n_h) + __compute_feature__(x_i, n1_x, n1_y, n1_w, n1_h)
X_feat[i, j] = int32(p1) - int32(n1)
return X_feat
@njit('int32[:, :](int32[:, :], uint16[:, :], uint8[:], float64[:])')
def train_weak_clf(X_feat: np.ndarray, X_feat_argsort: np.ndarray, y: np.ndarray, weights: np.ndarray) -> np.ndarray:
"""Train the weak classifiers on a given dataset (CPU version).
Args:
X_feat (np.ndarray): Feature images dataset.
X_feat_argsort (np.ndarray): Sorted indexes of the integrated features.
y (np.ndarray): Labels of the features.
weights (np.ndarray): Weights of the features.
X_feat (np.ndarray): Feature images dataset
X_feat_argsort (np.ndarray): Sorted indexes of the integrated features
y (np.ndarray): Labels of the features
weights (np.ndarray): Weights of the features
Returns:
np.ndarray: Trained weak classifiers.
np.ndarray: Trained weak classifiers
"""
total_pos, total_neg = weights[y == 1].sum(), weights[y == 0].sum()
@ -112,29 +71,85 @@ def train_weak_clf(X_feat: np.ndarray, X_feat_argsort: np.ndarray, y: np.ndarray
classifiers[i] = (best_threshold, best_polarity)
return classifiers
@njit('uint32(uint32[:, :], int16, int16, int16, int16)')
def __compute_feature__(ii: np.ndarray, x: int, y: int, w: int, h: int) -> int:
"""Compute a feature on an integrated image at a specific coordinate (CPU version).
Args:
ii (np.ndarray): Integrated image
x (int): X coordinate
y (int): Y coordinate
w (int): width of the feature
h (int): height of the feature
Returns:
int: Computed feature
"""
return ii[y + h, x + w] + ii[y, x] - ii[y + h, x] - ii[y, x + w]
@njit('int32[:, :](uint8[:, :, :, :], uint32[:, :, :])')
def apply_features(feats: np.ndarray, X_ii: np.ndarray) -> np.ndarray:
"""Apply the features on a integrated image dataset (CPU version).
Args:
feats (np.ndarray): Features to apply
X_ii (np.ndarray): Integrated image dataset
Returns:
np.ndarray: Applied features
"""
X_feat = np.empty((feats.shape[0], X_ii.shape[0]), dtype = np.int32)
for i, (p, n) in enumerate(tqdm_iter(feats, "Applying features")):
for j, x_i in enumerate(X_ii):
p_x, p_y, p_w, p_h = p[0]
p1_x, p1_y, p1_w, p1_h = p[1]
n_x, n_y, n_w, n_h = n[0]
n1_x, n1_y, n1_w, n1_h = n[1]
p1 = __compute_feature__(x_i, p_x, p_y, p_w, p_h) + __compute_feature__(x_i, p1_x, p1_y, p1_w, p1_h)
n1 = __compute_feature__(x_i, n_x, n_y, n_w, n_h) + __compute_feature__(x_i, n1_x, n1_y, n1_w, n1_h)
X_feat[i, j] = int32(p1) - int32(n1)
return X_feat
@njit('int32(int32[:], uint16[:], int32, int32)')
def as_partition(a: np.ndarray, indices: np.ndarray, l: int, h: int) -> int:
i = l - 1
j = l
for j in range(l, h + 1):
if a[indices[j]] < a[indices[h]]:
def _as_partition_(d_a: np.ndarray, d_indices: np.ndarray, low: int, high: int) -> int:
"""Partition of the argsort algorithm.
Args:
d_a (np.ndarray): Array on device to sort
d_indices (np.ndarray): Array of indices on device to write to
low (int): lower bound to sort
high (int): higher bound to sort
Returns:
int: Last index sorted
"""
i, j = low - 1, low
for j in range(low, high + 1):
if d_a[d_indices[j]] < d_a[d_indices[high]]:
i += 1
indices[i], indices[j] = indices[j], indices[i]
d_indices[i], d_indices[j] = d_indices[j], d_indices[i]
i += 1
indices[i], indices[j] = indices[j], indices[i]
d_indices[i], d_indices[j] = d_indices[j], d_indices[i]
return i
@njit('void(int32[:], uint16[:], int32, int32)')
def argsort_bounded(a: np.ndarray, indices: np.ndarray, l: int, h: int):
total = h - l + 1;
stack = np.empty((total,), dtype = np.int32)
stack[0] = l
stack[1] = h
top = 1;
def argsort_bounded(d_a: np.ndarray, d_indices: np.ndarray, low: int, high: int) -> None:
"""Perform an indirect sort of a given array within a given bound.
low = l
high = h
Args:
d_a (np.ndarray): Array to sort
d_indices (np.ndarray): Array of indices to write to
low (int): lower bound to sort
high (int): higher bound to sort
"""
total = high - low + 1
stack = np.empty((total,), dtype = np.int32)
stack[0] = low
stack[1] = high
top = 1
while top >= 0:
high = stack[top]
@ -143,24 +158,32 @@ def argsort_bounded(a: np.ndarray, indices: np.ndarray, l: int, h: int):
top -= 1
if low >= high:
break;
break
p = as_partition(a, indices, low, high);
p = _as_partition_(d_a, d_indices, low, high)
if p - 1 > low:
top += 1
stack[top] = low;
stack[top] = low
top += 1
stack[top] = p - 1;
stack[top] = p - 1
if p + 1 < high:
top += 1
stack[top] = p + 1;
stack[top] = p + 1
top += 1
stack[top] = high;
stack[top] = high
@njit('uint16[:, :](int32[:, :])')
def argsort(X_feat: np.ndarray) -> np.ndarray:
"""Perform an indirect sort of a given array.
Args:
X_feat (np.ndarray): Array to sort
Returns:
np.ndarray: Array of indices that sort the array
"""
indices = np.empty_like(X_feat, dtype = np.uint16)
indices[:, :] = np.arange(indices.shape[1])
for i in tqdm_iter(range(X_feat.shape[0]), "argsort"):

179
python/ViolaJonesGPU.py
View File

@ -12,10 +12,10 @@ def __scanCPU_3d__(X: np.ndarray) -> np.ndarray:
"""Prefix Sum (scan) of a given dataset.
Args:
X (np.ndarray): Dataset of images to apply sum.
X (np.ndarray): Dataset of images to apply sum
Returns:
np.ndarray: Scanned dataset of images.
np.ndarray: Scanned dataset of images
"""
for x in range(X.shape[0]):
for y in range(X.shape[1]):
@ -30,10 +30,10 @@ def __kernel_scan_3d__(n: int, j: int, d_inter: np.ndarray, d_a: np.ndarray) ->
"""GPU kernel used to do a parallel prefix sum (scan).
Args:
n (int):
j (int): [description]
d_inter (np.ndarray): [description]
d_a (np.ndarray): [description]
n (int): Number of width blocks
j (int): Temporary sum index
d_inter (np.ndarray): Temporary sums in device to add
d_a (np.ndarray): Dataset of images in device to apply sum
"""
x_coor, y_coor = cuda.grid(2)
@ -76,10 +76,10 @@ def __add_3d__(d_X: np.ndarray, d_s: np.ndarray, n: int, m: int) -> None:
"""GPU kernel for parallel sum.
Args:
d_X (np.ndarray): Dataset of images.
d_s (np.ndarray): Temporary sums to add.
n (int): Number of width blocks.
m (int): Height of a block.
d_X (np.ndarray): Dataset of images in device
d_s (np.ndarray): Temporary sums in device to add
n (int): Number of width blocks
m (int): Height of a block
"""
x_coor, y_coor = cuda.grid(2)
if x_coor < n and y_coor < m:
@ -91,10 +91,10 @@ def __scanGPU_3d__(X: np.ndarray) -> np.ndarray:
Read more: https://developer.nvidia.com/gpugems/gpugems3/part-vi-gpu-computing/chapter-39-parallel-prefix-sum-scan-cuda
Args:
X (np.ndarray): Dataset of images.
X (np.ndarray): Dataset of images
Returns:
np.ndarray: Scanned dataset of images.
np.ndarray: Scanned dataset of images
"""
k, height, n = X.shape
n_block_x, n_block_y = np.ceil(np.divide(X.shape[1:], NB_THREADS_2D)).astype(np.uint64)
@ -131,10 +131,10 @@ def __transpose_kernel__(d_X: np.ndarray, d_Xt: np.ndarray) -> None:
"""GPU kernel of the function __transpose_3d__.
Args:
d_X (np.ndarray): Dataset of images.
d_Xt(np.ndarray): Transposed dataset of images.
width (int): Width of each images in the dataset.
height (int): Height of each images in the dataset.
d_X (np.ndarray): Dataset of images in device
d_Xt(np.ndarray): Transposed dataset of images
width (int): Width of each images in the dataset
height (int): Height of each images in the dataset
"""
temp = cuda.shared.array(NB_THREADS_2D, dtype = uint32)
@ -152,10 +152,10 @@ def __transpose_3d__(X: np.ndarray) -> np.ndarray:
"""Transpose every images in the given dataset.
Args:
X (np.ndarray): Dataset of images.
X (np.ndarray): Dataset of images
Returns:
np.ndarray: Transposed dataset of images.
np.ndarray: Transposed dataset of images
"""
n_block_x, n_block_z = np.ceil(np.divide(X.shape[1:], NB_THREADS_2D)).astype(np.uint64)
d_X = cuda.to_device(X)
@ -167,10 +167,10 @@ def set_integral_image(X: np.ndarray) -> np.ndarray:
"""Transform the input images in integrated images (GPU version).
Args:
X (np.ndarray): Dataset of images.
X (np.ndarray): Dataset of images
Returns:
np.ndarray: Dataset of integrated images.
np.ndarray: Dataset of integrated images
"""
X = X.astype(np.uint32)
X = __scanGPU_3d__(X)
@ -184,13 +184,13 @@ def __train_weak_clf_kernel__(d_classifiers: np.ndarray, d_y: np.ndarray, d_X_fe
"""GPU kernel of the function train_weak_clf.
Args:
d_classifiers (np.ndarray): Weak classifiers to train.
d_y (np.ndarray): Labels of the features.
d_X_feat (np.ndarray): Feature images dataset.
d_X_feat_argsort (np.ndarray): Sorted indexes of the integrated features.
d_weights (np.ndarray): Weights of the features.
total_pos (float): Total of positive labels in the dataset.
total_neg (float): Total of negative labels in the dataset.
d_classifiers (np.ndarray): Weak classifiers to train
d_y (np.ndarray): Labels of the features
d_X_feat (np.ndarray): Feature images dataset
d_X_feat_argsort (np.ndarray): Sorted indexes of the integrated features
d_weights (np.ndarray): Weights of the features
total_pos (float): Total of positive labels in the dataset
total_neg (float): Total of negative labels in the dataset
"""
i = cuda.blockIdx.x * cuda.blockDim.x * cuda.blockDim.y * cuda.blockDim.z
i += cuda.threadIdx.x * cuda.blockDim.y * cuda.blockDim.z
@ -224,13 +224,13 @@ def train_weak_clf(X_feat: np.ndarray, X_feat_argsort: np.ndarray, y: np.ndarray
"""Train the weak classifiers on a given dataset (GPU version).
Args:
X_feat (np.ndarray): Feature images dataset.
X_feat_argsort (np.ndarray): Sorted indexes of the integrated features.
y (np.ndarray): Labels of the features.
weights (np.ndarray): Weights of the features.
X_feat (np.ndarray): Feature images dataset
X_feat_argsort (np.ndarray): Sorted indexes of the integrated features
y (np.ndarray): Labels of the features
weights (np.ndarray): Weights of the features
Returns:
np.ndarray: Trained weak classifiers.
np.ndarray: Trained weak classifiers
"""
total_pos, total_neg = weights[y == 1].sum(), weights[y == 0].sum()
d_classifiers = cuda.to_device(np.empty((X_feat.shape[0], 2), dtype = np.int32))
@ -247,14 +247,14 @@ def __compute_feature__(ii: np.ndarray, x: int, y: int, w: int, h: int) -> int:
"""Compute a feature on an integrated image at a specific coordinate (GPU version).
Args:
ii (np.ndarray): Integrated image.
x (int): X coordinate.
y (int): Y coordinate.
w (int): width of the feature.
h (int): height of the feature.
ii (np.ndarray): Integrated image
x (int): X coordinate
y (int): Y coordinate
w (int): width of the feature
h (int): height of the feature
Returns:
int: Computed feature.
int: Computed feature
"""
return ii[y + h, x + w] + ii[y, x] - ii[y + h, x] - ii[y, x + w]
@ -263,11 +263,11 @@ def __apply_feature_kernel__(X_feat: np.ndarray, feats: np.ndarray, X_ii: np.nda
"""GPU kernel of the function apply_features.
Args:
X_feat (np.ndarray): Feature images dataset.
feats (np.ndarray): Features to apply.
X_ii (np.ndarray): Integrated image dataset.
n (int): Number of features.
m (int): Number of images of the dataset.
X_feat (np.ndarray): Feature images dataset on device
feats (np.ndarray): Features on device to apply
X_ii (np.ndarray): Integrated image dataset on device
n (int): Number of features
m (int): Number of images of the dataset
"""
x, y = cuda.grid(2)
if x >= feats.shape[0] or y >= X_ii.shape[0]:
@ -288,11 +288,11 @@ def apply_features(feats: np.ndarray, X_ii: np.ndarray) -> np.ndarray:
"""Apply the features on a integrated image dataset (GPU version).
Args:
feats (np.ndarray): Features to apply.
X_ii (np.ndarray): Integrated image dataset.
feats (np.ndarray): Features to apply
X_ii (np.ndarray): Integrated image dataset
Returns:
np.ndarray: Applied features.
np.ndarray: Applied features
"""
d_X_feat = cuda.to_device(np.empty((feats.shape[0], X_ii.shape[0]), dtype = np.int32))
d_feats = cuda.to_device(feats)
@ -303,28 +303,44 @@ def apply_features(feats: np.ndarray, X_ii: np.ndarray) -> np.ndarray:
return d_X_feat.copy_to_host()
@cuda.jit('int32(int32[:], uint16[:], int32, int32)', device = True)
def as_partition(a: np.ndarray, indices: np.ndarray, l: int, h: int) -> int:
def _as_partition_(d_a: np.ndarray, d_indices: np.ndarray, l: int, h: int) -> int:
"""Partition of the argsort algorithm.
Args:
d_a (np.ndarray): Array on device to sort
d_indices (np.ndarray): Array of indices on device to write to
low (int): lower bound to sort
high (int): higher bound to sort
Returns:
int: Last index sorted
"""
i = l - 1
j = l
for j in range(l, h + 1):
if a[indices[j]] < a[indices[h]]:
if d_a[d_indices[j]] < d_a[d_indices[h]]:
i += 1
indices[i], indices[j] = indices[j], indices[i]
d_indices[i], d_indices[j] = d_indices[j], d_indices[i]
i += 1
indices[i], indices[j] = indices[j], indices[i]
d_indices[i], d_indices[j] = d_indices[j], d_indices[i]
return i
@cuda.jit('void(int32[:], uint16[:], int32, int32)', device = True)
def argsort_bounded(a: np.ndarray, indices: np.ndarray, l: int, h: int) -> None:
#total = h - l + 1;
stack = cuda.local.array(6977, int32)
stack[0] = l
stack[1] = h
top = 1;
def argsort_bounded(d_a: np.ndarray, d_indices: np.ndarray, low: int, high: int) -> None:
"""Perform an indirect sort of a given array within a given bound.
low = l
high = h
Args:
d_a (np.ndarray): Array on device to sort
d_indices (np.ndarray): Array of indices on device to write to
low (int): lower bound to sort
high (int): higher bound to sort
"""
#total = high - low + 1;
stack = cuda.local.array(6977, int32)
stack[0] = low
stack[1] = high
top = 1
while top >= 0:
high = stack[top]
@ -333,34 +349,49 @@ def argsort_bounded(a: np.ndarray, indices: np.ndarray, l: int, h: int) -> None:
top -= 1
if low >= high:
break;
break
p = as_partition(a, indices, low, high);
p = _as_partition_(d_a, d_indices, low, high)
if p - 1 > low:
top += 1
stack[top] = low;
stack[top] = low
top += 1
stack[top] = p - 1;
stack[top] = p - 1
if p + 1 < high:
top += 1
stack[top] = p + 1;
stack[top] = p + 1
top += 1
stack[top] = high;
stack[top] = high
@cuda.jit('void(int32[:, :], uint16[:, :])')
def argsort_flatter(X_feat: np.ndarray, indices: np.ndarray) -> None:
i = cuda.blockIdx.x * cuda.blockDim.x + cuda.threadIdx.x
if i < X_feat.shape[0]:
for j in range(indices.shape[1]):
indices[i, j] = j
argsort_bounded(X_feat[i], indices[i], 0, X_feat.shape[1] - 1)
def argsort_flatter(d_a: np.ndarray, d_indices: np.ndarray) -> None:
# TODO Finish doxygen
"""Cuda kernel where argsort is applied to every columns of a given 2D array.
def argsort(X_feat: np.ndarray) -> np.ndarray:
indices = np.empty_like(X_feat, dtype = np.uint16)
n_blocks = int(np.ceil(np.divide(X_feat.shape[0], NB_THREADS)))
d_X_feat = cuda.to_device(X_feat)
Args:
d_a (np.ndarray): Array in device to sort
d_indices (np.ndarray): Array of indices on device to write to
"""
i = cuda.blockIdx.x * cuda.blockDim.x + cuda.threadIdx.x
if i < d_a.shape[0]:
for j in range(d_indices.shape[1]):
d_indices[i, j] = j
argsort_bounded(d_a[i], d_indices[i], 0, d_a.shape[1] - 1)
def argsort(a: np.ndarray) -> np.ndarray:
"""Perform an indirect sort of a given array
Args:
a (np.ndarray): Array to sort
Returns:
np.ndarray: Array of indices that sort the array
"""
indices = np.empty_like(a, dtype = np.uint16)
n_blocks = int(np.ceil(np.divide(a.shape[0], NB_THREADS)))
d_X_feat = cuda.to_device(a)
d_indices = cuda.to_device(indices)
argsort_flatter[n_blocks, NB_THREADS](d_X_feat, d_indices)
cuda.synchronize()

11
python/activate.sh
View File

@ -3,9 +3,8 @@
# Exit if any of the command doesn't exit with code 0
set -e
EXEC_DIR=$1
test -z "$EXEC_DIR" && EXEC_DIR=..
VENV_PATH=$EXEC_DIR/python/venv
test -z "$EXEC_DIR" && EXEC_DIR=.
test -z "$VENV_PATH" && VENV_PATH="$EXEC_DIR/venv"
activate(){
if [ ! -d "$VENV_PATH" ]; then
@ -16,9 +15,9 @@ activate(){
echo 'Updating base pip packages'
python -m pip install -U setuptools pip
echo 'Installing requirements'
pip install -r "$EXEC_DIR"/python/requirements.txt
elif [ -f "$VENV_PATH"/Scripts/activate ]; then source "$VENV_PATH"/Scripts/activate
elif [ -f "$VENV_PATH"/bin/activate ]; then source "$VENV_PATH"/bin/activate
pip install -r requirements.txt
elif [ -f "$VENV_PATH"/Scripts/activate ]; then . "$VENV_PATH"/Scripts/activate
elif [ -f "$VENV_PATH"/bin/activate ]; then . "$VENV_PATH"/bin/activate
else
echo 'Python virtual environnement not detected'
exit 1

92
python/common.py
View File

@ -1,29 +1,29 @@
from toolbox import picke_multi_loader, format_time_ns, unit_test_argsort_2d
from toolbox import pickle_multi_loader, format_time_ns, unit_test_argsort_2d, header, footer, formatted_line, formatted_row
from typing import List, Tuple
from time import perf_counter_ns
from sys import stderr
import numpy as np
from config import OUT_DIR, DATA_DIR, __DEBUG
def unit_test(TS: List[int], labels: List[str] = ["CPU", "GPU", "PY", "PGPU"], tol: float = 1e-8) -> None:
def unit_test(TS: List[int], labels: List[str] = ['CPU', 'GPU', 'PY', 'PGPU'], tol: float = 1e-8) -> None:
"""Test if the each result is equals to other devices.
Given ViolaJones is a deterministic algorithm, the results no matter the device should be the same
Given ViolaJones is a fully deterministic algorithm. The results, regardless the device, should be the same
(given the floating point fluctuations), this function check this assertion.
Args:
TS (List[int]): Number of trained weak classifiers.
labels (List[str], optional): List of the trained device names. Defaults to ["CPU", "GPU", "PY", "PGPU"] (see config.py for more info).
tol (float, optional): Float difference tolerance. Defaults to 1e-8.
TS (List[int]): Number of trained weak classifiers
labels (List[str], optional): List of the trained device names. Defaults to ['CPU', 'GPU', 'PY', 'PGPU'] (see config.py for more info)
tol (float, optional): Float difference tolerance. Defaults to 1e-8
"""
if len(labels) < 2:
return print("Not enough devices to test")
return print('Not enough devices to test')
print(f"\n| {'Unit testing':<37} | {'Test state':<10} | {'Time spent (ns)':<18} | {'Formatted time spent':<29} |")
print(f"|{'-'*39}|{'-'*12}|{'-'*20}|{'-'*31}|")
unit_gaps = [37, -10, -18, 29]
header(unit_gaps, ['Unit testing', 'Test state', 'Time spent (ns)', 'Formatted time spent'])
fnc_s = perf_counter_ns()
n_total = 0
n_success = 0
unit_timestamp = perf_counter_ns()
n_total, n_success = 0, 0
def test_fnc(title, fnc):
nonlocal n_total, n_success
@ -32,96 +32,102 @@ def unit_test(TS: List[int], labels: List[str] = ["CPU", "GPU", "PY", "PGPU"], t
state = fnc()
e = perf_counter_ns() - s
if state:
print(f"| {title:<37} | {'Passed':>10} | {e:>18,} | {format_time_ns(e):<29} |")
formatted_row(unit_gaps, [title, 'Passed', f'{e:,}', format_time_ns(e)])
n_success += 1
else:
print(f"| {title:<37} | {'Failed':>10} | {e:>18,} | {format_time_ns(e):<29} |")
formatted_row(unit_gaps, [title, 'Failed', f'{e:,}', format_time_ns(e)])
for set_name in ["train", "test"]:
for filename in ["ii", "feat"]:
title = f"X_{set_name}_{filename}"
print(f"{filename}...", end = "\r")
bs = picke_multi_loader([f"{title}_{label}" for label in labels], OUT_DIR)
for set_name in ['train', 'test']:
for filename in ['ii', 'feat']:
title = f'X_{set_name}_{filename}'
print(f'{filename}...', file = stderr, end = '\r')
bs = pickle_multi_loader([f'{title}_{label}' for label in labels], OUT_DIR)
for i, (b1, l1) in enumerate(zip(bs, labels)):
if b1 is None:
if __DEBUG:
print(f"| {title:<22} - {l1:<12} | {'Skipped':>10} | {'None':>18} | {'None':<29} |")
formatted_row(unit_gaps, [f'{title:<22} - {l1:<12}', 'Skipped', 'None', 'None'])
continue
for j, (b2, l2) in enumerate(zip(bs, labels)):
if i >= j:
continue
if b2 is None:
if __DEBUG:
print(f"| {title:<22} - {l1:<4} vs {l2:<4} | {'Skipped':>10} | {'None':>18} | {'None':<29} |")
formatted_row(unit_gaps, [f'{title:<22} - {l1:<4} vs {l2:<4}', 'Skipped', 'None', 'None'])
continue
test_fnc(f"{title:<22} - {l1:<4} vs {l2:<4}", lambda: np.abs(b1 - b2).mean() < tol)
test_fnc(f'{title:<22} - {l1:<4} vs {l2:<4}', lambda: np.abs(b1 - b2).mean() < tol)
title = f"X_{set_name}_feat_argsort"
print(f"Loading {title}...", end = "\r")
title = f'X_{set_name}_feat_argsort'
print(f'Loading {title}...', file = stderr, end = '\r')
feat = None
bs = []
for label in labels:
if feat is None:
feat_tmp = picke_multi_loader([f"X_{set_name}_feat_{label}"], OUT_DIR)[0]
feat_tmp = pickle_multi_loader([f'X_{set_name}_feat_{label}'], OUT_DIR)[0]
if feat_tmp is not None:
feat = feat_tmp
bs.append(picke_multi_loader([f"{title}_{label}"], OUT_DIR)[0])
bs.append(pickle_multi_loader([f'{title}_{label}'], OUT_DIR)[0])
for i, (b1, l1) in enumerate(zip(bs, labels)):
if b1 is None:
if __DEBUG:
print(f"| {title:<22} - {l1:<12} | {'Skipped':>10} | {'None':>18} | {'None':<29} |")
formatted_row(unit_gaps, [f'{title:<22} - {l1:<12}', 'Skipped', 'None', 'None'])
continue
if feat is not None:
test_fnc(f"{title:<22} - {l1:<4} argsort", lambda: unit_test_argsort_2d(feat, b1))
test_fnc(f'{title:<22} - {l1:<4} argsort', lambda: unit_test_argsort_2d(feat, b1))
for j, (b2, l2) in enumerate(zip(bs, labels)):
if i >= j:
continue
if b2 is None:
if __DEBUG:
print(f"| {title:<22} - {l1:<4} vs {l2:<4} | {'Skipped':>10} | {'None':>18} | {'None':<29} |")
formatted_row(unit_gaps, [f'{title:<22} - {l1:<4} vs {l2:<4}', 'Skipped', 'None', 'None'])
continue
test_fnc(f"{title:<22} - {l1:<4} vs {l2:<4}", lambda: np.abs(b1 - b2).mean() < tol)
test_fnc(f'{title:<22} - {l1:<4} vs {l2:<4}', lambda: np.abs(b1 - b2).mean() < tol)
for T in TS:
for filename in ["alphas", "final_classifiers"]:
print(f"{filename}_{T}...", end = "\r")
bs = picke_multi_loader([f"{filename}_{T}_{label}" for label in labels])
for filename in ['alphas', 'final_classifiers']:
print(f'{filename}_{T}...', file = stderr, end = '\r')
bs = pickle_multi_loader([f'{filename}_{T}_{label}' for label in labels])
for i, (b1, l1) in enumerate(zip(bs, labels)):
if b1 is None:
if __DEBUG:
print(f"| {filename + '_' + str(T):<22} - {l1:<12} | {'Skipped':>10} | {'None':>18} | {'None':<29} |")
formatted_row(unit_gaps, [f"{filename + '_' + str(T):<22} - {l1:<12}", 'Skipped', 'None', 'None'])
continue
for j, (b2, l2) in enumerate(zip(bs, labels)):
if i >= j:
continue
if b2 is None:
if __DEBUG:
print(f"| {filename + '_' + str(T):<22} - {l1:<4} vs {l2:<4} | {'Skipped':>10} | {'None':>18} | {'None':<29} |")
formatted_row(unit_gaps, [f"{filename + '_' + str(T):<22} - {l1:<4} vs {l2:<4}", 'Skipped', 'None', 'None'])
continue
test_fnc(f"{filename + '_' + str(T):<22} - {l1:<4} vs {l2:<4}", lambda: np.abs(b1 - b2).mean() < tol)
print(f"|{'-'*39}|{'-'*12}|{'-'*20}|{'-'*31}|")
e = perf_counter_ns() - fnc_s
print(f"| {'Unit testing summary':<37} | {str(n_success) + '/' + str(n_total):>10} | {e:>18,} | {format_time_ns(e):<29} |")
time_spent = perf_counter_ns() - unit_timestamp
if n_total == 0:
formatted_row(unit_gaps, ['Unit testing summary', 'No files', f'{time_spent:,}', format_time_ns(time_spent)])
else:
formatted_line(unit_gaps, '├', '┼', '─', '┤')
formatted_row(unit_gaps, ['Unit testing summary', f'{n_success}/{n_total}', f'{time_spent:,}', format_time_ns(time_spent)])
footer(unit_gaps)
def load_datasets(data_dir: str = DATA_DIR) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
"""Load the datasets.
Args:
data_dir (str, optional): [description]. Defaults to DATA_DIR (see config.py).
data_dir (str, optional): [description]. Defaults to DATA_DIR (see config.py)
Returns:
Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: [description]
Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: X_train, y_train, X_test, y_test
"""
bytes_to_int_list = lambda b: list(map(int, b.rstrip().split(" ")))
bytes_to_int_list = lambda b: list(map(int, b.rstrip().split(' ')))
def load(set_name: str) -> np.ndarray:
with open(f"{data_dir}/{set_name}.bin", "r") as f:
with open(f'{data_dir}/{set_name}.bin', 'r') as f:
shape = bytes_to_int_list(f.readline())
return np.asarray(bytes_to_int_list(f.readline()), dtype = np.uint8).reshape(shape)
return load("X_train"), load("y_train"), load("X_test"), load("y_test")
return load('X_train'), load('y_train'), load('X_test'), load('y_test')

7
python/convert_dataset.py
View File

@ -5,6 +5,9 @@ from sys import argv
import numpy as np
from os import path, listdir
# Induce determinism
np.random.seed(133742)
# Makes the "leave" argument default to False
tqdm = partial(tqdm, leave = False)
@ -42,8 +45,8 @@ def __main__(data_path: str) -> None:
y.append(y_i)
X, y = np.asarray(X), np.asarray(y)
# idx = np.random.permutation(y.shape[0])
# X, y = X[idx], y[idx]
idx = np.random.permutation(y.shape[0])
X, y = X[idx], y[idx]
for org, s in tqdm(zip("Xy", [X, y]), desc = f"Writing {set_name}"):
with open(f"{data_path}/{org}_{set_name}.bin", "w") as out:

17
python/decorators.py
View File

@ -2,6 +2,14 @@ from typing import Callable, Iterable, Union, Any
from tqdm import tqdm
def njit(f: Union[Callable, str] = None, *args, **kwargs) -> Callable:
"""Wrapper for optional numba's njit decorator
Args:
f (Union[Callable, str], optional): Function to wrap with numba. Defaults to None.
Returns:
Callable: Wrapped function.
"""
def decorator(func: Callable) -> Any:
return func
@ -10,4 +18,13 @@ def njit(f: Union[Callable, str] = None, *args, **kwargs) -> Callable:
return decorator
def tqdm_iter(iter: Iterable, desc: str):
"""Wrapper for optional tqdm iterator progress bar.
Args:
iter (Iterable): Object to iterate over.
desc (str): Description written to stdout.
Returns:
_type_: Wrapped iterator.
"""
return tqdm(iter, leave = False, desc = desc)

106
python/projet.py
View File

@ -2,14 +2,15 @@
# Author: @saundersp
from ViolaJones import train_viola_jones, classify_viola_jones
from toolbox import state_saver, picke_multi_loader, format_time_ns, benchmark_function, unit_test_argsort_2d
from toolbox_unit_test import format_time_ns_test
#from toolbox import state_saver, pickle_multi_loader, format_time_ns, benchmark_function, unit_test_argsort_2d
from toolbox import state_saver, format_time_ns, benchmark_function, unit_test_argsort_2d
from toolbox import header, footer, formatted_row, formatted_line
from toolbox_unit_test import format_time_test, format_time_ns_test
from sklearn.metrics import accuracy_score, f1_score, confusion_matrix
from sklearn.feature_selection import SelectPercentile, f_classif
#from sklearn.feature_selection import SelectPercentile, f_classif
from common import load_datasets, unit_test
from ViolaJones import build_features, get_best_anova_features
from typing import Tuple
from ViolaJones import build_features # , get_best_anova_features
from typing import Tuple, List
from time import perf_counter_ns
from os import makedirs
import numpy as np
@ -28,35 +29,42 @@ else:
from ViolaJonesCPU import apply_features, set_integral_image, argsort
label = 'CPU' if COMPILE_WITH_C else 'PY'
def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
"""Load the dataset, calculate features and integral images, apply features to images and calculate argsort of the featured images.
def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
"""Execute the preprocessing phase
The preprocessing phase consist of the following steps :
- Load the dataset
- Calculate features
- Calculate integral images
- Apply features to images
- Calculate argsort of the featured images.
Returns:
Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: X_train_feat, X_train_feat_argsort, y_train, X_test_feat, y_test
Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: Tuple containing in order : training features, training features sorted indexes, training labels, testing features, testing labels
"""
# Creating state saver folders if they don't exist already
if SAVE_STATE:
for folder_name in ["models", "out"]:
for folder_name in ['models', 'out']:
makedirs(folder_name, exist_ok = True)
preproc_timestamp = perf_counter_ns()
preproc_gaps = [49, -18, 29]
header(['Preprocessing', 'Time spent (ns)', 'Formatted time spent'], preproc_gaps)
header(preproc_gaps, ['Preprocessing', 'Time spent (ns)', 'Formatted time spent'])
X_train, y_train, X_test, y_test = state_saver('Loading sets', preproc_gaps[0], ['X_train', 'y_train', 'X_test', 'y_test'],
load_datasets, FORCE_REDO, SAVE_STATE)
load_datasets, FORCE_REDO, SAVE_STATE)
if __DEBUG:
print("X_train")
print('X_train')
print(X_train.shape)
print(X_train[IDX_INSPECT])
print("X_test")
print('X_test')
print(X_test.shape)
print(X_test[IDX_INSPECT])
print("y_train")
print('y_train')
print(y_train.shape)
print(y_train[IDX_INSPECT: IDX_INSPECT + IDX_INSPECT_OFFSET])
print("y_test")
print('y_test')
print(y_test.shape)
print(y_test[IDX_INSPECT: IDX_INSPECT + IDX_INSPECT_OFFSET])
@ -64,7 +72,7 @@ def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
FORCE_REDO, SAVE_STATE)
if __DEBUG:
print("feats")
print('feats')
print(feats.shape)
print(feats[IDX_INSPECT].ravel())
@ -74,10 +82,10 @@ def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
lambda: set_integral_image(X_test), FORCE_REDO, SAVE_STATE)
if __DEBUG:
print("X_train_ii")
print('X_train_ii')
print(X_train_ii.shape)
print(X_train_ii[IDX_INSPECT])
print("X_test_ii")
print('X_test_ii')
print(X_test_ii.shape)
print(X_test_ii[IDX_INSPECT])
@ -88,25 +96,25 @@ def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
del X_train_ii, X_test_ii, feats
if __DEBUG:
print("X_train_feat")
print('X_train_feat')
print(X_train_feat.shape)
print(X_train_feat[IDX_INSPECT, : IDX_INSPECT_OFFSET])
print("X_test_feat")
print('X_test_feat')
print(X_test_feat.shape)
print(X_test_feat[IDX_INSPECT, : IDX_INSPECT_OFFSET])
#indices = state_saver("Selecting best features training set", "indices", force_redo = True, save_state = SAVE_STATE,
#indices = state_saver('Selecting best features training set', 'indices', force_redo = FORCE_REDO, save_state = SAVE_STATE,
# fnc = lambda: SelectPercentile(f_classif, percentile = 10).fit(X_train_feat.T, y_train).get_support(indices = True))
#indices = state_saver("Selecting best features training set", "indices", force_redo = FORCE_REDO, save_state = SAVE_STATE,
#indices = state_saver('Selecting best features training set', 'indices', force_redo = FORCE_REDO, save_state = SAVE_STATE,
# fnc = lambda: get_best_anova_features(X_train_feat, y_train))
#indices = benchmark_function("Selecting best features (manual)", lambda: get_best_anova_features(X_train_feat, y_train))
#indices = benchmark_function('Selecting best features (manual)', lambda: get_best_anova_features(X_train_feat, y_train))
#if __DEBUG:
# print("indices")
# print('indices')
# print(indices.shape)
# print(indices[IDX_INSPECT: IDX_INSPECT + IDX_INSPECT_OFFSET])
# assert indices.shape[0] == indices_new.shape[0], f"Indices length not equal : {indices.shape} != {indices_new.shape}"
# assert (eq := indices == indices_new).all(), f"Indices not equal : {eq.sum() / indices.shape[0]}"
# assert indices.shape[0] == indices_new.shape[0], f'Indices length not equal : {indices.shape} != {indices_new.shape}'
# assert (eq := indices == indices_new).all(), f'Indices not equal : {eq.sum() / indices.shape[0]}'
# X_train_feat, X_test_feat = X_train_feat[indices], X_test_feat[indices]
@ -114,19 +122,20 @@ def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
lambda: argsort(X_train_feat), FORCE_REDO, SAVE_STATE)
if __DEBUG:
print("X_train_feat_argsort")
print('X_train_feat_argsort')
print(X_train_feat_argsort.shape)
print(X_train_feat_argsort[IDX_INSPECT, : IDX_INSPECT_OFFSET])
benchmark_function('Arg unit test', preproc_gaps[0], lambda: unit_test_argsort_2d(X_train_feat, X_train_feat_argsort))
X_test_feat_argsort = state_saver(f"Precalculating testing set argsort ({label})", f"X_test_feat_argsort_{label}",
X_test_feat_argsort = state_saver(f'Precalculating testing set argsort ({label})', preproc_gaps[0], f'X_test_feat_argsort_{label}',
lambda: argsort(X_test_feat), FORCE_REDO, SAVE_STATE)
if __DEBUG:
print("X_test_feat_argsort")
print('X_test_feat_argsort')
print(X_test_feat_argsort.shape)
print(X_test_feat_argsort[IDX_INSPECT, : IDX_INSPECT_OFFSET])
benchmark_function("Arg unit test", lambda: unit_test_argsort_2d(X_test_feat, X_test_feat_argsort))
benchmark_function('Arg unit test', lambda: unit_test_argsort_2d(X_test_feat, X_test_feat_argsort))
time_spent = perf_counter_ns() - preproc_timestamp
formatted_line(preproc_gaps, '├', '┼', '─', '┤')
formatted_row(preproc_gaps, ['Preprocessing summary', f'{time_spent:,}', format_time_ns(time_spent)])
@ -138,16 +147,17 @@ def train(X_train_feat: np.ndarray, X_train_feat_argsort: np.ndarray, y_train: n
"""Train the weak classifiers.
Args:
X_train (np.ndarray): Training images.
X_train_feat_argsort (np.ndarray): Sorted indexes of the training images features.
y_train (np.ndarray): Training labels.
X_train (np.ndarray): Training images
X_train_feat_argsort (np.ndarray): Sorted indexes of the training images features
y_train (np.ndarray): Training labels
Returns: List of trained models
Returns:
List[np.ndarray]: List of trained models
"""
training_timestamp = perf_counter_ns()
training_gaps = [26, -18, 29]
header(['Training', 'Time spent (ns)', 'Formatted time spent'], training_gaps)
header(training_gaps, ['Training', 'Time spent (ns)', 'Formatted time spent'])
models = []
for T in TS:
@ -157,9 +167,9 @@ def train(X_train_feat: np.ndarray, X_train_feat_argsort: np.ndarray, y_train: n
models.append([alphas, final_classifiers])
if __DEBUG:
print("alphas")
print('alphas')
print(alphas)
print("final_classifiers")
print('final_classifiers')
print(final_classifiers)
time_spent = perf_counter_ns() - training_timestamp
@ -173,15 +183,15 @@ def testing_and_evaluating(models: List[np.ndarray], X_train_feat: np.ndarray, y
"""Benchmark the trained classifiers on the training and testing sets.
Args:
models (List[np.ndarray]): List of trained models.
X_train_feat (np.ndarray): Training features.
y_train (np.ndarray): Training labels.
X_test_feat (np.ndarray): Testing features.
y_test (np.ndarray): Testing labels.
models (List[np.ndarray]): List of trained models
X_train_feat (np.ndarray): Training features
y_train (np.ndarray): Training labels
X_test_feat (np.ndarray): Testing features
y_test (np.ndarray): Testing labels
"""
testing_gaps = [26, -19, 24, -19, 24]
header(['Testing', 'Time spent (ns) (E)', 'Formatted time spent (E)', 'Time spent (ns) (T)', 'Formatted time spent (T)'], testing_gaps)
header(testing_gaps, ['Testing', 'Time spent (ns) (E)', 'Formatted time spent (E)', 'Time spent (ns) (T)', 'Formatted time spent (T)'])
performances = []
total_train_timestamp = 0
@ -213,7 +223,7 @@ def testing_and_evaluating(models: List[np.ndarray], X_train_feat: np.ndarray, y
footer(testing_gaps)
evaluating_gaps = [19, 7, 6, 6, 6, 7, 6, 6, 6]
header(['Evaluating', 'ACC (E)', 'F1 (E)', 'FN (E)', 'FP (E)', 'ACC (T)', 'F1 (T)', 'FN (T)', 'FP (T)'], evaluating_gaps)
header(evaluating_gaps, ['Evaluating', 'ACC (E)', 'F1 (E)', 'FN (E)', 'FP (E)', 'ACC (T)', 'F1 (T)', 'FN (T)', 'FP (T)'])
for T, (e_acc, e_f1, e_FN, e_FP, t_acc, t_f1, t_FN, t_FP) in zip(TS, performances):
print(f'│ ViolaJones T = {T:<4} │ {e_acc:>7.2%} │ {e_f1:>6.2f} │ {e_FN:>6,} │ {e_FP:>6,}', end = ' │ ')
@ -224,7 +234,7 @@ def testing_and_evaluating(models: List[np.ndarray], X_train_feat: np.ndarray, y
def main() -> None:
unit_timestamp = perf_counter_ns()
unit_gaps = [27, -18, 29]
header(['Unit testing', 'Time spent (ns)', 'Formatted time spent'], unit_gaps)
header(unit_gaps, ['Unit testing', 'Time spent (ns)', 'Formatted time spent'])
benchmark_function('testing format_time', unit_gaps[0], format_time_test)
benchmark_function('testing format_time_ns', unit_gaps[0], format_time_ns_test)
time_spent = perf_counter_ns() - unit_timestamp
@ -235,12 +245,12 @@ def main() -> None:
X_train_feat, X_train_feat_argsort, y_train, X_test_feat, y_test = preprocessing()
models = train(X_train_feat, X_train_feat_argsort, y_train)
# X_train_feat, X_test_feat = picke_multi_loader([f"X_train_feat_{label}", f"X_test_feat_{label}"], OUT_DIR)
# indices = picke_multi_loader(["indices"], OUT_DIR)[0]
# X_train_feat, X_test_feat = pickle_multi_loader([f'X_train_feat_{label}', f'X_test_feat_{label}'], OUT_DIR)
# indices = pickle_multi_loader(['indices'], OUT_DIR)[0]
# X_train_feat, X_test_feat = X_train_feat[indices], X_test_feat[indices]
testing_and_evaluating(models, X_train_feat, y_train, X_test_feat, y_test)
unit_test(TS)
if __name__ == "__main__":
if __name__ == '__main__':
main()

189
python/test.py
View File

@ -1,189 +0,0 @@
import numpy as np
from numba import cuda, config, njit
config.CUDA_LOW_OCCUPANCY_WARNINGS = 0
#import matplotlib.pyplot as plt
from tqdm import tqdm
from time import perf_counter_ns
from toolbox import format_time_ns
from pickle import load, dump
from sys import argv
def get(a):
with open(f"{a}.pkl", 'rb') as f:
return load(f)
def save(a, name) -> None:
with open(name, 'wb') as f:
dump(a, f)
def diff(folder, a, label1, label2):
af, bf = get(f"{folder}/{a}_{label1}"), get(f"{folder}/{a}_{label2}")
#print(af)
#print(bf)
print((af - bf).mean())
if __name__ == "__main__":
if len(argv) == 5:
diff(argv[1], argv[4], argv[2], argv[3])
def py_mean(a, b):
s = 0.0
for a_i, b_i in zip(a, b):
s += a_i * b_i
return s / a.shape[0]
def np_mean(a, b):
return np.mean(a * b)
@njit('float64(float64[:], float64[:])', fastmath = True, nogil = True)
def nb_mean(a, b):
return np.mean(a * b)
@njit('float64(float64[:], float64[:])', fastmath = True, nogil = True)
def nb_mean_loop(a, b):
s = 0.0
for a_i, b_i in zip(a, b):
s += a_i * b_i
return s / a.shape[0]
@cuda.jit('void(float64[:], float64[:], float64[:])', fastmath = True)
def cuda_mean_kernel(r, a, b):
s = 0.0
for a_i, b_i in zip(a, b):
s += a_i * b_i
r[0] = s / a.shape[0]
def cuda_mean(a, b):
r = cuda.to_device(np.empty(1, dtype = np.float64))
d_a = cuda.to_device(a)
d_b = cuda.to_device(b)
cuda_mean_kernel[1, 1](r, d_a, d_b)
return r.copy_to_host()[0]
def test_and_compare(labels, fncs, a, b):
m = []
for fnc in tqdm(fncs, leave = False, desc = "Calculating..."):
s = perf_counter_ns()
m.append([fnc(a, b), perf_counter_ns() - s])
print("Results:")
[print(f"\t{label:<10} {m_i:<20} {format_time_ns(time_i)}") for ((m_i, time_i), label) in zip(m, labels)]
print("Comparaison:")
for i, (m_i, label_i) in enumerate(zip(m, labels)):
for j, (m_j, label_j) in enumerate(zip(m, labels)):
if i >= j:
continue
print(f"\t{label_i:<10} vs {label_j:<10} - {abs(m_i[0] - m_j[0])}")
if __name__ == "__main__":
np.set_printoptions(linewidth = 10000, threshold = 1000)
N = int(2**20)
labels = ["Python", "Numpy", "Numba", "Numba loop", "CUDA"]
fncs = [py_mean, np_mean, nb_mean, nb_mean_loop, cuda_mean]
print(f"RANDOM for N={N}")
total_size = (2 * 8 * N)
print(f"Size = {total_size} B")
print(f"Size = {total_size // 1024} kB")
print(f"Size = {total_size // 1024 // 1024} MB")
print(f"Size = {total_size // 1024 // 1024 // 1024} GB")
a, b = np.random.rand(N).astype(np.float64), np.random.rand(N).astype(np.float64)
test_and_compare(labels, fncs, a, b)
del a, b
print(f"\nDETERMINSTIC for N={N}")
total_size = (2 * 8 * N) + (8 * N)
print(f"Size = {total_size} B")
print(f"Size = {total_size // 1024} kB")
print(f"Size = {total_size // 1024 // 1024} MB")
print(f"Size = {total_size // 1024 // 1024 // 1024} GB")
mask = np.arange(N, dtype = np.uint64)
a = np.ones(N, dtype = np.float64)
a[mask < N//2] = 0.1
del mask
b = np.ones(N, dtype = np.float64)
test_and_compare(labels, fncs, a, b)
del a, b
#from ViolaJonesGPU import argsort as argsort_GPU
#from ViolaJonesCPU import argsort as argsort_CPU
#from toolbox import unit_test_argsort_2d, benchmark_function
#labels = ["Numpy", "Numba", "CUDA"]
#a = np.random.randint(2**12, size = (2**20, 2**8), dtype = np.int32)
#m = [benchmark_function(f"Argsort {label}", lambda: f(np.copy(a))) for (label, f) in zip(labels, [
# lambda a: np.argsort(a).astype(np.uint16), argsort_CPU, argsort_GPU
#])]
#for i, (m_i, label_i) in enumerate(zip(m, labels)):
# #for j, (m_j, label_j) in enumerate(zip(m, labels)):
# # if i >= j:
# # continue
# # print(f"\t{label_i:<10} vs {label_j:<10} - {(m_i == m_j).mean()}")
# benchmark_function(f"Unit test {label_i}", lambda: unit_test_argsort_2d(a, m_i))
#for i in tqdm(range(X.shape[0]), leave = False, desc = "Extract image"):
# x = X[i]
# y = Y[i]
# fig = plt.figure()
# plt.imshow(x, cmap = 'gray')
# plt.savefig(f"imgs/{y}/{i}.png")
# plt.close(fig)
#def extract_FD(Xy):
# X_c, Y_c = [], []
# for x,y in Xy:
# X_c.append(x)
# Y_c.append(y)
# X_c = np.asarray(X_c)
# Y_c = np.asarray(Y_c)
# return X_c, Y_c
#X_train, y_train = get('out/X_train'), get('out/y_train')
#X_test, y_test = get('out/X_test'), get('out/y_test')
#X_train, y_train = extract_FD(get('/home/_aspil0w/git/FaceDetection/training'))
#X_test, y_test = extract_FD(get('/home/_aspil0w/git/FaceDetection/test'))
#save(X_train, 'out/X_train'), save(y_train, 'out/y_train')
#save(X_test, 'out/X_test'), save(y_test, 'out/y_test')
#print(X_train.shape, X_train_org.shape, X_train.shape == X_train_org.shape)
#print((X_train == X_train_org).mean())
#print(y_train.shape, y_train_org.shape, y_train.shape == y_train_org.shape)
#print((y_train == y_train_org).mean())
#print(X_test.shape, X_test_org.shape, X_test.shape == X_test_org.shape)
#print((X_test == X_test_org).mean())
#print(y_test.shape, y_test_org.shape, y_test.shape == y_test_org.shape)
#print((y_test == y_test_org).mean())
#@njit('uint16[:](uint8[:, :, :], uint8[:, :, :])')
#def arg_find(X, X_org):
# arg = np.empty(X.shape[0], dtype = np.uint16)
# for i, x in enumerate(X_org):
# found = False
# for j, x_org in enumerate(X):
# if np.all(x == x_org):
# arg[i] = j
# found = True
# break
# assert found, "Image not found"
# return arg
#print("Arg find results train")
#arg_train = arg_find(X_train, X_train_org)
#print((X_train[arg_train] == X_train_org).mean())
#print((y_train[arg_train] == y_train_org).mean())
#print("Arg find results test")
#arg_test = arg_find(X_test, X_test_org)
#print((X_test[arg_test] == X_test_org).mean())
#print((y_test[arg_test] == y_test_org).mean())
#for i in tqdm(range(X_c.shape[0]), leave = False, desc = "Extract image"):
# x = X_c[i]
# y = Y_c[i]
# fig = plt.figure()
# plt.imshow(x, cmap = 'gray')
# plt.savefig(f"imgs2/{y}/{i}.png")
# plt.close(fig)

130
python/toolbox.py
View File

@ -1,6 +1,5 @@
from typing import Any, Callable, List, Union, Final
from time import perf_counter_ns
from numba import njit
import numpy as np
from sys import stderr
import pickle
@ -8,70 +7,122 @@ import os
from config import MODEL_DIR, OUT_DIR
from decorators import njit
time_formats: Final = ["ns", "µs", "ms", "s", "m", "h", "j", "w", "M", "y", "c"]
def formatted_row(gaps: list[int], titles: list[str], separator: str = '│') -> None:
"""Print a formatted row of titles with of gaps seperated by a separator.
Args:
gaps: List of size gaps
titles: List of titles
separator: Separator character between each gap
"""
for gap, title in zip(gaps, titles):
print(f"{separator} {title:{'>' if gap < 0 else '<'}{abs(gap)}} ", end = '')
print(separator)
def formatted_line(gaps: list[int], right: str, middle: str, separator: str, left: str) -> None:
print(right, end = '')
def formatted_line(gaps: list[int], left: str, middle: str, separator: str, right: str) -> None:
"""Print a formatted line of repeated characters.
Args:
gaps: List of size gaps
left: Character on the left
middle: Character between each separator
separator: Separator character between each gap
right: Character on the right
"""
print(left, end = '')
last_gap = len(gaps) - 1
for i, gap in enumerate(gaps):
print(f'{separator * (abs(gap) + 2)}', end = '')
if i != last_gap:
print(middle, end = '')
print(left)
print(right)
def header(titles: list[str], gaps: list[int]) -> None:
def header(gaps: list[int], titles: list[str]) -> None:
"""Print a formatted header with the given titles and sizes.
Args:
gaps: List of size gaps
titles: List of titles
"""
formatted_line(gaps, '┌', '┬', '─', '┐')
formatted_row(gaps, titles)
formatted_line(gaps, '├', '┼', '─', '┤')
def footer(gaps: list[int]) -> None:
"""Print a formatted fooder with the given sizes
Args:
gaps: List of size gaps
"""
formatted_line(gaps, '└', '┴', '─', '┘')
time_formats: Final = ['ns', 'µs', 'ms', 's', 'm', 'h', 'j', 'w', 'M', 'y', 'c']
time_numbers: Final = np.array([1, 1e3, 1e6, 1e9, 6e10, 36e11, 864e11, 6048e11, 26784e11, 31536e12, 31536e14], dtype = np.uint64)
@njit('str(uint64)')
def format_time_ns(time: int) -> str:
"""Format the time in nanoseconds in human readable format.
Args:
time (int): Time in nanoseconds.
time (int): Time in nanoseconds
Returns:
str: The formatted human readable string.
str: The formatted human readable string
"""
assert time >= 0, "Incorrect time stamp"
assert time >= 0, 'Incorrect time stamp'
if time == 0:
return "0ns"
return '0ns'
s = ""
s = ''
for i in range(time_numbers.shape[0])[::-1]:
if time >= time_numbers[i]:
res = int(time // time_numbers[i])
time = time % time_numbers[i]
s += f"{res}{time_formats[i]} "
s += f'{res}{time_formats[i]} '
assert time == 0, "Leftover in formatting time !"
assert time == 0, 'Leftover in formatting time !'
return s.rstrip()
def picke_multi_loader(filenames: List[str], save_dir: str = MODEL_DIR) -> List[Any]:
@njit('str(uint64)')
def format_time(time: int) -> str:
"""Format the time in seconds in human readable format.
Args:
time (int): Time in seconds
Returns:
str: The formatted human readable string
"""
assert time >= 0, 'Incorrect time stamp'
if time == 0:
return '0s'
s = ''
for i in range(3, time_numbers.shape[0])[::-1]:
time_number = time_numbers[i] / int(1e9)
if time >= time_number:
res = int(time // time_number)
time = time % time_number
s += f'{res}{time_formats[i]} '
assert time == 0, 'Leftover in formatting time !'
return s.rstrip()
def pickle_multi_loader(filenames: List[str], save_dir: str = MODEL_DIR) -> List[Any]:
"""Load multiple pickle data files.
Args:
filenames (List[str]): List of all the filename to load.
save_dir (str, optional): Path of the files to load. Defaults to MODELS_DIR (see config.py).
filenames (List[str]): List of all the filename to load
save_dir (str, optional): Path of the files to load. Defaults to MODELS_DIR (see config.py)
Returns:
List[Any]. List of loaded pickle data files.
List[Any]. List of loaded pickle data files
"""
b = []
for f in filenames:
filepath = f"{save_dir}/{f}.pkl"
filepath = f'{save_dir}/{f}.pkl'
if os.path.exists(filepath):
with open(filepath, "rb") as filebyte:
b.append(pickle.load(filebyte))
with open(filepath, 'rb') as file_bytes:
b.append(pickle.load(file_bytes))
else:
b.append(None)
return b
@ -80,11 +131,11 @@ def benchmark_function(step_name: str, column_width: int, fnc: Callable) -> Any:
"""Benchmark a function and display the result of stdout.
Args:
step_name (str): Name of the function to call.
fnc (Callable): Function to call.
step_name (str): Name of the function to call
fnc (Callable): Function to call
Returns:
Any: Result of the function.
Any: Result of the function
"""
print(f'{step_name}...', file = stderr, end = '\r')
s = perf_counter_ns()
@ -98,34 +149,34 @@ def state_saver(step_name: str, column_width: int, filename: Union[str, List[str
"""Either execute a function then saves the result or load the already existing result.
Args:
step_name (str): Name of the function to call.
filename (Union[str, List[str]]): Name or list of names of the filenames where the result(s) are saved.
fnc ([type]): Function to call.
force_redo (bool, optional): Recall the function even if the result(s) is already saved. Defaults to False.
save_dir (str, optional): Path of the directory to save the result(s). Defaults to OUT_DIR (see config.py).
step_name (str): Name of the function to call
filename (Union[str, List[str]]): Name or list of names of the filenames where the result(s) are saved
fnc ([type]): Function to call
force_redo (bool, optional): Recall the function even if the result(s) is already saved. Defaults to False
save_dir (str, optional): Path of the directory to save the result(s). Defaults to OUT_DIR (see config.py)
Returns:
Any: The result(s) of the called function
"""
if isinstance(filename, str):
if not os.path.exists(f"{save_dir}/{filename}.pkl") or force_redo:
if not os.path.exists(f'{save_dir}/{filename}.pkl') or force_redo:
b = benchmark_function(step_name, column_width, fnc)
if save_state:
with open(f"{save_dir}/{filename}.pkl", 'wb') as f:
print(f'Saving results of {step_name}', file = stderr, end = '\r')
with open(f'{save_dir}/{filename}.pkl', 'wb') as f:
pickle.dump(b, f)
print(' ' * 100, file = stderr, end = '\r')
return b
else:
with open(f"{save_dir}/{filename}.pkl", "rb") as f:
print(f'Loading results of {step_name}', file = stderr, end = '\r')
with open(f'{save_dir}/{filename}.pkl', 'rb') as f:
res = pickle.load(f)
print(f"│ {step_name:<{column_width}} │ {'None':>18} │ {'loaded saved state':<29} │")
return res
elif isinstance(filename, list):
abs = False
for fn in filename:
if not os.path.exists(f"{save_dir}/{fn}.pkl"):
if not os.path.exists(f'{save_dir}/{fn}.pkl'):
abs = True
break
if abs or force_redo:
@ -133,7 +184,7 @@ def state_saver(step_name: str, column_width: int, filename: Union[str, List[str
if save_state:
print(f'Saving results of {step_name}', file = stderr, end = '\r')
for bi, fnI in zip(b, filename):
with open(f"{save_dir}/{fnI}.pkl", 'wb') as f:
with open(f'{save_dir}/{fnI}.pkl', 'wb') as f:
pickle.dump(bi, f)
print(' ' * 100, file = stderr, end = '\r')
return b
@ -142,15 +193,24 @@ def state_saver(step_name: str, column_width: int, filename: Union[str, List[str
b = []
print(f'Loading results of {step_name}', file = stderr, end = '\r')
for fn in filename:
with open(f"{save_dir}/{fn}.pkl", "rb") as f:
with open(f'{save_dir}/{fn}.pkl', 'rb') as f:
b.append(pickle.load(f))
print(' ' * 100, file = stderr, end = '\r')
return b
else:
assert False, f"Incompatible filename type = {type(filename)}"
assert False, f'Incompatible filename type = {type(filename)}'
@njit('boolean(int32[:, :], uint16[:, :])')
def unit_test_argsort_2d(arr: np.ndarray, indices: np.ndarray) -> bool:
"""Test if a given array of indices sort a given array.
Args:
arr (np.ndarray): Array of data
indices (np.ndarray): Indices that sort arr
Returns:
bool: Success of the test
"""
n = indices.shape[0]
total = indices.shape[0] * indices.shape[1]
for i, sub_indices in enumerate(indices):

183
python/toolbox_unit_test.py
View File

@ -1,67 +1,132 @@
from typing import Any
from toolbox import format_time_ns
from toolbox import format_time, format_time_ns
def Assert(name: str, expected: Any, result: Any):
"""Test if a given result is equal of the expected one and log result
Args:
name (str): name of the unit test
expected (Any): expected result of the function call
result (Any): result of the function
"""
if expected != result:
print(f"For test name {name} Expected '{expected}' but got '{result}' instead")
assert False
def format_time_ns_test() -> None:
# https://en.wikipedia.org/wiki/Unit_of_time
Assert("format_time_ns null", "0ns", format_time_ns(0));
Assert("format_time_ns nanosecond", "1ns", format_time_ns(1));
Assert("format_time_ns shake", "10ns", format_time_ns(10));
Assert("format_time_ns microsecond", "1µs", format_time_ns(int(1e3)));
Assert("format_time_ns millisecond", "1ms", format_time_ns(int(1e6)));
Assert("format_time_ns centisecond", "10ms", format_time_ns(int(1e7)));
Assert("format_time_ns decisecond", "100ms", format_time_ns(int(1e8)));
Assert("format_time_ns second", "1s", format_time_ns(int(1e9)));
Assert("format_time_ns decasecond", "10s", format_time_ns(int(1e10)));
Assert("format_time_ns minute", "1m", format_time_ns(int(6e10)));
Assert("format_time_ns milliday", "1m 26s 400ms", format_time_ns(int(864e8)));
Assert("format_time_ns hectosecond", "1m 40s", format_time_ns(int(1e11)));
Assert("format_time_ns kilosecond", "16m 40s", format_time_ns(int(1e12)));
Assert("format_time_ns hour", "1h", format_time_ns(int(36e11)));
Assert("format_time_ns day", "1j", format_time_ns(int(864e11)));
Assert("format_time_ns week/sennight", "1w", format_time_ns(int(6048e11)));
Assert("format_time_ns megasecond", "1w 4j 13h 46m 40s", format_time_ns(int(1e15)));
Assert("format_time_ns fortnight", "2w", format_time_ns(int(12096e11)));
Assert("format_time_ns lunar month (draconitic)", "3w 6j 5h 5m 35s 800ms", format_time_ns(int(23511358e8)));
Assert("format_time_ns lunar month (tropical)", "3w 6j 7h 43m 4s 700ms", format_time_ns(int(23605847e8)));
Assert("format_time_ns lunar month (sidereal)", "3w 6j 7h 43m 11s 600ms", format_time_ns(int(23605916e8)));
Assert("format_time_ns lunar month (anomalistic)", "3w 6j 13h 18m 33s 200ms", format_time_ns(int(23807132e8)));
Assert("format_time_ns lunar month (synodic)", "4w 1j 12h 44m 2s 900ms", format_time_ns(int(25514429e8)));
Assert("format_time_ns month", "1M", format_time_ns(int(26784e11)));
Assert("format_time_ns quarantine", "1M 1w 2j", format_time_ns(int(3456e12)));
Assert("format_time_ns semester", "4M 2j", format_time_ns(int(108864e11)));
Assert("format_time_ns lunar year", "11M 1w 6j 8h 52m 48s", format_time_ns(int(30617568e9)));
Assert("format_time_ns year", "1y", format_time_ns(int(31536e12)));
Assert("format_time_ns tropical year", "1y 5h 48m 45s 216ms", format_time_ns(int(31556925216e6)));
Assert("format_time_ns gregorian year", "1y 5h 49m 12s", format_time_ns(int(31556952e9)));
Assert("format_time_ns sidereal year", "1y 6h 9m 9s 763ms 545µs 600ns", format_time_ns(int(315581497635456e2)));
Assert("format_time_ns leap year", "1y 1j", format_time_ns(int(316224e11)));
Assert("format_time_ns olympiad", "4y", format_time_ns(int(126144e12)));
Assert("format_time_ns lusturm", "5y", format_time_ns(int(15768e13)));
Assert("format_time_ns decade", "10y", format_time_ns(int(31536e13)));
Assert("format_time_ns indiction", "15y", format_time_ns(int(47304e13)));
Assert("format_time_ns score", "20y", format_time_ns(int(63072e13)));
Assert("format_time_ns gigasecond", "31y 8M 1w 4j 1h 46m 40s", format_time_ns(int(1e18)));
Assert("format_time_ns jubilee", "50y", format_time_ns(int(15768e14)));
Assert("format_time_ns century", "1c", format_time_ns(int(31536e14)));
Assert("format_time_ns millennium", "10c", format_time_ns(int(31536e15)));
Assert("format_time_ns age", "257c 72y", format_time_ns(int(812745792e12)));
Assert("format_time_ns terasecond", "3170c 97y 10M 3w 4j 17h 46m 40s", format_time_ns(int(1e22)));
Assert("format_time_ns megaannum", "10000c", format_time_ns(int(31536e18)));
# Cannot use number bigger than currently supported ISO Python
#Assert("format_time_ns petasecond", "317097c 91y 11M 2w 4j 1h 46m 40s", format_time_ns(int(1e24)));
#Assert("format_time_ns galactic year", "2300000c", format_time_ns(int(725328e19)));
#Assert("format_time_ns eon", "10000000c", format_time_ns(int(31536e21)));
#Assert("format_time_ns kalpa", "43200000c", format_time_ns(int(13623552e19)));
#Assert("format_time_ns exasecond", "317097919c 83y 9M 1h 46m 40s", format_time_ns(int(1e27)));
#Assert("format_time_ns zettasecond", "", format_time_ns(int(1e30)));
#Assert("format_time_ns yottasecond", "", format_time_ns(int(1e33)));
#Assert("format_time_ns ronnasecond", "", format_time_ns(int(1e36)));
#Assert("format_time_ns quettasecond", "", format_time_ns(int(1e39)));
# uint64_t_MAX == 2**64 == 18446744073709551615I64u == -1
Assert("format_time_ns max", "5c 84y 11M 2j 23h 34m 33s 709ms 551µs 615ns", format_time_ns(2**64 - 1))
def format_time_test() -> None:
"""Test suite for the format_time output
See https://en.wikipedia.org/wiki/Unit_of_time for details
"""
Assert("format_time null", "0s", format_time(0))
Assert("format_time second", "1s", format_time(1))
Assert("format_time decasecond", "10s", format_time(10))
Assert("format_time minute", "1m", format_time(60))
Assert("format_time milliday", "1m 26s", format_time(86)) # missing 0.4s due to precision
Assert("format_time hectosecond", "1m 40s", format_time(100))
Assert("format_time kilosecond", "16m 40s", format_time(int(1e3)))
Assert("format_time hour", "1h", format_time(3600))
Assert("format_time day", "1j", format_time(86400))
Assert("format_time week/sennight", "1w", format_time(604800))
Assert("format_time megasecond", "1w 4j 13h 46m 40s", format_time(int(1e6)))
Assert("format_time fortnight", "2w", format_time(1209600))
Assert("format_time lunar month (draconitic)", "3w 6j 5h 5m 35s", format_time(2351135)) # missing 0.8 due to precision
Assert("format_time lunar month (tropical)", "3w 6j 7h 43m 4s", format_time(2360584)) # missing 0.7 due to precision
Assert("format_time lunar month (sidereal)", "3w 6j 7h 43m 11s", format_time(2360591)) # missing 0.6 to precision
Assert("format_time lunar month (anomalistic)", "3w 6j 13h 18m 33s", format_time(2380713)) # missing 0.2 due to precision
Assert("format_time lunar month (synodic)", "4w 1j 12h 44m 2s", format_time(2551442)) # missing 0.9 due to precision
Assert("format_time month", "1M", format_time(2678400))
Assert("format_time quarantine", "1M 1w 2j", format_time(int(3456e3)))
Assert("format_time semester", "4M 2j", format_time(10886400))
Assert("format_time lunar year", "11M 1w 6j 8h 52m 48s", format_time(30617568))
Assert("format_time year", "1y", format_time(int(31536e3)))
Assert("format_time tropical year", "1y 5h 48m 45s", format_time(31556925)) # missing 0.216 due to precision
Assert("format_time gregorian year", "1y 5h 49m 12s", format_time(31556952))
Assert("format_time sidereal year", "1y 6h 9m 9s", format_time(31558149)) # missing 0.7635456 due to precision
Assert("format_time leap year", "1y 1j", format_time(31622400))
Assert("format_time olympiad", "4y", format_time(int(126144e3)))
Assert("format_time lusturm", "5y", format_time(int(15768e4)))
Assert("format_time decade", "10y", format_time(int(31536e4)))
Assert("format_time indiction", "15y", format_time(int(47304e4)))
Assert("format_time score", "20y", format_time(int(63072e4)))
Assert("format_time gigasecond", "31y 8M 1w 4j 1h 46m 40s", format_time(int(1e9)))
Assert("format_time jubilee", "50y", format_time(int(15768e5)))
Assert("format_time century", "1c", format_time(int(31536e5)))
Assert("format_time millennium", "10c", format_time(int(31536e6)))
Assert("format_time age", "257c 72y", format_time(int(812745792e3)))
Assert("format_time terasecond", "3170c 97y 10M 3w 4j 17h 46m 40s", format_time(int(1e13)))
Assert("format_time megaannum", "10000c", format_time(int(31536e9)))
Assert("format_time petasecond", "317097c 91y 11M 2w 4j 1h 46m 40s", format_time(int(1e15)))
Assert("format_time galactic year", "2300000c", format_time(int(725328e10)))
Assert("format_time eon", "10000000c", format_time(int(31536e12)))
Assert("format_time kalpa", "43200000c", format_time(int(13623552e10)))
Assert("format_time exasecond", "317097919c 83y 9M 1h 46m 40s", format_time(int(1e18)))
# Cannot use number bigger than currently supported ISO Python
#Assert("format_time zettasecond", "", format_time(1e21))
#Assert("format_time yottasecond", "", format_time(1e24))
#Assert("format_time ronnasecond", "", format_time(1e27))
#Assert("format_time quettasecond", "", format_time(1e30))
# uint64_t_MAX == 2**64 == 18446744073709551615 == -1
Assert("format_time max", "5849424173c 55y 3w 5j 7h 16s", format_time(int(2**64 - 1)))
def format_time_ns_test() -> None:
"""Test suite for the format_time_ns output
See https://en.wikipedia.org/wiki/Unit_of_time for details
"""
Assert("format_time_ns null", "0ns", format_time_ns(0))
Assert("format_time_ns nanosecond", "1ns", format_time_ns(1))
Assert("format_time_ns shake", "10ns", format_time_ns(10))
Assert("format_time_ns microsecond", "1µs", format_time_ns(int(1e3)))
Assert("format_time_ns millisecond", "1ms", format_time_ns(int(1e6)))
Assert("format_time_ns centisecond", "10ms", format_time_ns(int(1e7)))
Assert("format_time_ns decisecond", "100ms", format_time_ns(int(1e8)))
Assert("format_time_ns second", "1s", format_time_ns(int(1e9)))
Assert("format_time_ns decasecond", "10s", format_time_ns(int(1e10)))
Assert("format_time_ns minute", "1m", format_time_ns(int(6e10)))
Assert("format_time_ns milliday", "1m 26s 400ms", format_time_ns(int(864e8)))
Assert("format_time_ns hectosecond", "1m 40s", format_time_ns(int(1e11)))
Assert("format_time_ns kilosecond", "16m 40s", format_time_ns(int(1e12)))
Assert("format_time_ns hour", "1h", format_time_ns(int(36e11)))
Assert("format_time_ns day", "1j", format_time_ns(int(864e11)))
Assert("format_time_ns week/sennight", "1w", format_time_ns(int(6048e11)))
Assert("format_time_ns megasecond", "1w 4j 13h 46m 40s", format_time_ns(int(1e15)))
Assert("format_time_ns fortnight", "2w", format_time_ns(int(12096e11)))
Assert("format_time_ns lunar month (draconitic)", "3w 6j 5h 5m 35s 800ms", format_time_ns(int(23511358e8)))
Assert("format_time_ns lunar month (tropical)", "3w 6j 7h 43m 4s 700ms", format_time_ns(int(23605847e8)))
Assert("format_time_ns lunar month (sidereal)", "3w 6j 7h 43m 11s 600ms", format_time_ns(int(23605916e8)))
Assert("format_time_ns lunar month (anomalistic)", "3w 6j 13h 18m 33s 200ms", format_time_ns(int(23807132e8)))
Assert("format_time_ns lunar month (synodic)", "4w 1j 12h 44m 2s 900ms", format_time_ns(int(25514429e8)))
Assert("format_time_ns month", "1M", format_time_ns(int(26784e11)))
Assert("format_time_ns quarantine", "1M 1w 2j", format_time_ns(int(3456e12)))
Assert("format_time_ns semester", "4M 2j", format_time_ns(int(108864e11)))
Assert("format_time_ns lunar year", "11M 1w 6j 8h 52m 48s", format_time_ns(int(30617568e9)))
Assert("format_time_ns year", "1y", format_time_ns(int(31536e12)))
Assert("format_time_ns tropical year", "1y 5h 48m 45s 216ms", format_time_ns(int(31556925216e6)))
Assert("format_time_ns gregorian year", "1y 5h 49m 12s", format_time_ns(int(31556952e9)))
Assert("format_time_ns sidereal year", "1y 6h 9m 9s 763ms 545µs 600ns", format_time_ns(int(315581497635456e2)))
Assert("format_time_ns leap year", "1y 1j", format_time_ns(int(316224e11)))
Assert("format_time_ns olympiad", "4y", format_time_ns(int(126144e12)))
Assert("format_time_ns lusturm", "5y", format_time_ns(int(15768e13)))
Assert("format_time_ns decade", "10y", format_time_ns(int(31536e13)))
Assert("format_time_ns indiction", "15y", format_time_ns(int(47304e13)))
Assert("format_time_ns score", "20y", format_time_ns(int(63072e13)))
Assert("format_time_ns gigasecond", "31y 8M 1w 4j 1h 46m 40s", format_time_ns(int(1e18)))
Assert("format_time_ns jubilee", "50y", format_time_ns(int(15768e14)))
Assert("format_time_ns century", "1c", format_time_ns(int(31536e14)))
Assert("format_time_ns millennium", "10c", format_time_ns(int(31536e15)))
Assert("format_time_ns age", "257c 72y", format_time_ns(int(812745792e12)))
Assert("format_time_ns terasecond", "3170c 97y 10M 3w 4j 17h 46m 40s", format_time_ns(int(1e22)))
Assert("format_time_ns megaannum", "10000c", format_time_ns(int(31536e18)))
# Cannot use number bigger than currently supported ISO Python
# Assert("format_time_ns petasecond", "317097c 91y 11M 2w 4j 1h 46m 40s", format_time_ns(int(1e24)))
# Assert("format_time_ns galactic year", "2300000c", format_time_ns(int(725328e19)))
# Assert("format_time_ns eon", "10000000c", format_time_ns(int(31536e21)))
# Assert("format_time_ns kalpa", "43200000c", format_time_ns(int(13623552e19)))
# Assert("format_time_ns exasecond", "317097919c 83y 9M 1h 46m 40s", format_time_ns(int(1e27)))
# Assert("format_time_ns zettasecond", "", format_time_ns(int(1e30)))
# Assert("format_time_ns yottasecond", "", format_time_ns(int(1e33)))
# Assert("format_time_ns ronnasecond", "", format_time_ns(int(1e36)))
# Assert("format_time_ns quettasecond", "", format_time_ns(int(1e39)))
# uint64_t_MAX == 2**64 == 18446744073709551615 == -1
Assert("format_time_ns max", "5c 84y 11M 2j 23h 34m 33s 709ms 551µs 615ns", format_time_ns(2**64 - 1))