shuf: avoid integer overflow on huge inputs

* gl/lib/randperm.c: Include <stdckdint.h>.
(randperm_bound): Return SIZE_MAX if the multiplication overflows.
Do not overflow when converting bit count to byte count.
This commit is contained in:
Paul Eggert
2024-08-03 22:59:12 -07:00
parent bfbb3ec7f7
commit 1ea7255f8b

View File

@@ -23,6 +23,7 @@
#include <limits.h>
#include <stdbit.h>
#include <stdckdint.h>
#include <stdint.h>
#include <stdlib.h>
@@ -39,13 +40,15 @@ randperm_bound (size_t h, size_t n)
{
/* Upper bound on number of bits needed to generate the first number
of the permutation. */
uintmax_t lg_n = stdc_bit_width (n) + 1;
unsigned int lg_n = stdc_bit_width (n) + 1;
/* Upper bound on number of bits needed to generated the first H elements. */
uintmax_t ar = lg_n * h;
/* Upper bound on number of bits needed to generate the first H elements. */
uintmax_t ar;
if (ckd_mul (&ar, lg_n, h))
return SIZE_MAX;
/* Convert the bit count to a byte count. */
size_t bound = (ar + CHAR_BIT - 1) / CHAR_BIT;
size_t bound = ar / CHAR_BIT + (ar % CHAR_BIT != 0);
return bound;
}