libosmogsm: Allow auth API caller to specify RES length

There are 3G algorithms which support different lengths of RES values
(4, 8, 16 byte).  For MILENAGE, we never really had to bother, as
the 4-byte RES is simply the first 4 bytes of the 8-byte RES.

However, for TUAK, the expected RES length is an input parameter to
the Keccak crypto functions, so the result of all parameters (including
CK, IK, ...) will be completely different for RES length 4 than RES
length 8.

So let's permit the caller of the osmocom auth API to specify the
requested RES length via the osmo_auth_vector.res_len parameter.

For backwards compatibility of callers of the old osmo_auth_gen_vec/
osmo_auth_gen_vec_auts API: Always force the res_len to 8 in this case,
which was the hard-coded length before this patch.

Change-Id: Ic662843fbe8b5c58e4af39ea630ad5ac13fd6bef
diff --git a/include/osmocom/crypt/auth.h b/include/osmocom/crypt/auth.h
index 2833ed3..dfec885 100644
--- a/include/osmocom/crypt/auth.h
+++ b/include/osmocom/crypt/auth.h
@@ -91,7 +91,7 @@
 	uint8_t ck[OSMO_A5_MAX_KEY_LEN_BYTES];		/*!< ciphering key */
 	uint8_t ik[OSMO_A5_MAX_KEY_LEN_BYTES];		/*!< integrity key */
 	uint8_t res[16];	/*!< authentication result */
-	uint8_t res_len;	/*!< length (in bytes) of res: 8..16 bytes */
+	uint8_t res_len;	/*!< length (in bytes) of res: 4..16 bytes */
 	uint8_t kc[8];		/*!< Kc for GSM encryption (A5) */
 	uint8_t sres[4];	/*!< authentication result for GSM */
 	uint32_t auth_types;	/*!< bitmask of OSMO_AUTH_TYPE_* */
diff --git a/src/gsm/auth_core.c b/src/gsm/auth_core.c
index 421ecee..2b6f35a 100644
--- a/src/gsm/auth_core.c
+++ b/src/gsm/auth_core.c
@@ -167,7 +167,7 @@
 }
 
 /*! Generate authentication vector
- *  \param[out] vec Generated authentication vector
+ *  \param[out] vec Generated authentication vector. See below!
  *  \param[in] aud Subscriber-specific key material
  *  \param[in] _rand Random challenge to be used
  *  \returns 0 on success, negative error on failure
@@ -176,7 +176,12 @@
  * computing authentication triples/quintuples based on the permanent
  * subscriber data and a random value.  The result is what is forwarded
  * by the AUC via HLR and VLR to the MSC which will then be able to
- * invoke authentication with the MS
+ * invoke authentication with the MS.
+ *
+ * Contrary to the older osmo_auth_gen_vec(), the caller must specify
+ * the desired RES length in the vec->res_len field prior to calling
+ * this function.  The requested length must match the capabilities of
+ * the chosen algorithm (e.g. 4/8 for MILENAGE).
  */
 int osmo_auth_gen_vec2(struct osmo_auth_vector *vec,
 		       struct osmo_sub_auth_data2 *aud,
@@ -216,6 +221,12 @@
 	struct osmo_sub_auth_data2 aud2;
 	int rc;
 
+	if (aud->type == OSMO_AUTH_TYPE_UMTS) {
+		/* old API callers are not expected to initialize this struct field,
+		 * and always expect an 8-byte RES value */
+		vec->res_len = 8;
+	}
+
 	rc = auth_data2auth_data2(&aud2, aud);
 	if (rc < 0)
 		return rc;
@@ -228,7 +239,7 @@
 }
 
 /*! Generate authentication vector and re-sync sequence
- *  \param[out] vec Generated authentication vector
+ *  \param[out] vec Generated authentication vector. See below!
  *  \param[in] aud Subscriber-specific key material
  *  \param[in] auts AUTS value sent by the SIM/MS
  *  \param[in] rand_auts RAND value sent by the SIM/MS
@@ -241,6 +252,11 @@
  * AUTS and RAND values returned by the SIM/MS.  This special variant is
  * needed if the sequence numbers between MS and AUC have for some
  * reason become different.
+ *
+ * Contrary to the older osmo_auth_gen_vec_auts(), the caller must specify
+ * the desired RES length in the vec->res_len field prior to calling
+ * this function.  The requested length must match the capabilities of
+ * the chosen algorithm (e.g. 4/8 for MILENAGE).
  */
 int osmo_auth_gen_vec_auts2(struct osmo_auth_vector *vec,
 			    struct osmo_sub_auth_data2 *aud,
@@ -285,6 +301,12 @@
 	struct osmo_sub_auth_data2 aud2;
 	int rc;
 
+	if (aud->type == OSMO_AUTH_TYPE_UMTS) {
+		/* old API callers are not expected to initialize this struct field,
+		 * and always expect an 8-byte RES value */
+		vec->res_len = 8;
+	}
+
 	rc = auth_data2auth_data2(&aud2, aud);
 	if (rc < 0)
 		return rc;
diff --git a/src/gsm/auth_milenage.c b/src/gsm/auth_milenage.c
index 9e94293..a439b15 100644
--- a/src/gsm/auth_milenage.c
+++ b/src/gsm/auth_milenage.c
@@ -62,6 +62,8 @@
 		return -EINVAL;
 	if (aud->u.umts.opc_len != 16)
 		return -EINVAL;
+	if (vec->res_len != 4 && vec->res_len != 8)
+		return -EINVAL;
 
 	opc = gen_opc_if_needed(aud, gen_opc);
 	if (!opc)
@@ -135,7 +137,7 @@
 	milenage_generate(opc, aud->u.umts.amf, aud->u.umts.k,
 			  sqn, _rand,
 			  vec->autn, vec->ik, vec->ck, vec->res, &res_len);
-	vec->res_len = res_len;
+
 	rc = gsm_milenage(opc, aud->u.umts.k, _rand, vec->sres, vec->kc);
 	if (rc < 0)
 		return rc;
diff --git a/tests/osmo-auc-gen/osmo-auc-gen_test.ok b/tests/osmo-auc-gen/osmo-auc-gen_test.ok
index 3c41f41..2da0b02 100644
--- a/tests/osmo-auc-gen/osmo-auc-gen_test.ok
+++ b/tests/osmo-auc-gen/osmo-auc-gen_test.ok
@@ -1,7 +1,7 @@
 
 
 > osmo-auc-gen -3 -a milenage -r 6a61050765caa32c90371370e5d6dc2d -k 1dc4f974325cce611e54f516dc1fec56 -o 2a48162ff3edca4adf0b7b5e527d6c16 -s 0
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 RAND:	6a61050765caa32c90371370e5d6dc2d
@@ -18,7 +18,7 @@
 
 
 > osmo-auc-gen -3 -a milenage -r 6a61050765caa32c90371370e5d6dc2d -k 1dc4f974325cce611e54f516dc1fec56 -o 2a48162ff3edca4adf0b7b5e527d6c16 -s 1
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 RAND:	6a61050765caa32c90371370e5d6dc2d
@@ -35,7 +35,7 @@
 
 
 > osmo-auc-gen -3 -a milenage -r 6a61050765caa32c90371370e5d6dc2d -k 1dc4f974325cce611e54f516dc1fec56 -o 2a48162ff3edca4adf0b7b5e527d6c16 -s 23
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 RAND:	6a61050765caa32c90371370e5d6dc2d
@@ -52,7 +52,7 @@
 
 
 > osmo-auc-gen -3 -a milenage -r 1dc4f974325cce611e54f516dc1fec56 -k 2a48162ff3edca4adf0b7b5e527d6c16 -o 6a61050765caa32c90371370e5d6dc2d -s 42
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 RAND:	1dc4f974325cce611e54f516dc1fec56
@@ -69,7 +69,7 @@
 
 
 > osmo-auc-gen -3 -a milenage -r 2a48162ff3edca4adf0b7b5e527d6c16 -k 6a61050765caa32c90371370e5d6dc2d -o 1dc4f974325cce611e54f516dc1fec56 -s 99
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 RAND:	2a48162ff3edca4adf0b7b5e527d6c16
@@ -86,7 +86,7 @@
 
 
 > osmo-auc-gen -3 -a milenage -r 6a61050765caa32c90371370e5d6dc2d -k 2a48162ff3edca4adf0b7b5e527d6c16 -o 1dc4f974325cce611e54f516dc1fec56 -s 281474976710655
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 RAND:	6a61050765caa32c90371370e5d6dc2d
@@ -103,7 +103,7 @@
 
 
 > osmo-auc-gen -3 -a milenage -r 39fa2f4e3d523d8619a73b4f65c3e14d -k EB215756028D60E3275E613320AEC880 -o FB2A3D1B360F599ABAB99DB8669F8308 -A 979498b1f72d3e28c59fa2e72f9c
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 RAND:	39fa2f4e3d523d8619a73b4f65c3e14d
@@ -121,7 +121,7 @@
 
 
 > osmo-auc-gen -3 -a milenage -r 39fa2f4e3d523d8619a73b4f65c3e14d -k EB215756028D60E3275E613320AEC880 -o FB2A3D1B360F599ABAB99DB8669F8308 -A 979498b1f72d3e28c59fa2e72f9c --ind 5
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 RAND:	39fa2f4e3d523d8619a73b4f65c3e14d
@@ -139,7 +139,7 @@
 
 
 > osmo-auc-gen -3 -a milenage -r 39fa2f4e3d523d8619a73b4f65c3e14d -k EB215756028D60E3275E613320AEC880 -o FB2A3D1B360F599ABAB99DB8669F8308 -A 979498b1f72d3e28c59fa2e72f9c --ind 23
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 RAND:	39fa2f4e3d523d8619a73b4f65c3e14d
@@ -157,7 +157,7 @@
 
 
 > osmo-auc-gen -3 -a milenage -r 39fa2f4e3d523d8619a73b4f65c3e14d -k EB215756028D60E3275E613320AEC880 -o FB2A3D1B360F599ABAB99DB8669F8308 -A 979498b1f72d3e28c59fa2e72f9c --ind 31
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 RAND:	39fa2f4e3d523d8619a73b4f65c3e14d
@@ -175,7 +175,7 @@
 
 
 > osmo-auc-gen -3 -a milenage -r 39fa2f4e3d523d8619a73b4f65c3e14d -k EB215756028D60E3275E613320AEC880 -o FB2A3D1B360F599ABAB99DB8669F8308 -A 979498b1f72d3e28c59fa2e72f9c --ind-len 0
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 RAND:	39fa2f4e3d523d8619a73b4f65c3e14d
@@ -193,7 +193,7 @@
 
 
 > osmo-auc-gen -3 -a milenage -r 39fa2f4e3d523d8619a73b4f65c3e14d -k EB215756028D60E3275E613320AEC880 -o FB2A3D1B360F599ABAB99DB8669F8308 -A 979498b1f72d3e28c59fa2e72f9c --ind-len 1
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 RAND:	39fa2f4e3d523d8619a73b4f65c3e14d
@@ -211,7 +211,7 @@
 
 
 > osmo-auc-gen -3 -a milenage -r 39fa2f4e3d523d8619a73b4f65c3e14d -k EB215756028D60E3275E613320AEC880 -o FB2A3D1B360F599ABAB99DB8669F8308 -A 979498b1f72d3e28c59fa2e72f9c --ind-len 1 --ind 1
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 RAND:	39fa2f4e3d523d8619a73b4f65c3e14d
@@ -229,7 +229,7 @@
 
 
 > osmo-auc-gen -3 -a milenage -r 39fa2f4e3d523d8619a73b4f65c3e14d -k EB215756028D60E3275E613320AEC880 -o FB2A3D1B360F599ABAB99DB8669F8308 -A 979498b1f72d3e28c59fa2e72f9c --ind-len 8
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 RAND:	39fa2f4e3d523d8619a73b4f65c3e14d
@@ -247,7 +247,7 @@
 
 
 > osmo-auc-gen -3 -a milenage -r 39fa2f4e3d523d8619a73b4f65c3e14d -k EB215756028D60E3275E613320AEC880 -o FB2A3D1B360F599ABAB99DB8669F8308 -A 979498b1f72d3e28c59fa2e72f9c --ind-len 8 --ind 1
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 RAND:	39fa2f4e3d523d8619a73b4f65c3e14d
@@ -266,27 +266,27 @@
 
 expecting error:
 > osmo-auc-gen -3 -a milenage -r 39fa2f4e3d523d8619a73b4f65c3e14d -k EB215756028D60E3275E613320AEC880 -o FB2A3D1B360F599ABAB99DB8669F8308 -A 979498b1f72d3e28c59fa2e72f9c --ind -1
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 
 
 expecting error:
 > osmo-auc-gen -3 -a milenage -r 39fa2f4e3d523d8619a73b4f65c3e14d -k EB215756028D60E3275E613320AEC880 -o FB2A3D1B360F599ABAB99DB8669F8308 -A 979498b1f72d3e28c59fa2e72f9c --ind 32
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 
 
 expecting error:
 > osmo-auc-gen -3 -a milenage -r 39fa2f4e3d523d8619a73b4f65c3e14d -k EB215756028D60E3275E613320AEC880 -o FB2A3D1B360F599ABAB99DB8669F8308 -A 979498b1f72d3e28c59fa2e72f9c --ind 42
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
 
 
 expecting error:
 > osmo-auc-gen -3 -a milenage -r 39fa2f4e3d523d8619a73b4f65c3e14d -k EB215756028D60E3275E613320AEC880 -o FB2A3D1B360F599ABAB99DB8669F8308 -A 979498b1f72d3e28c59fa2e72f9c --ind-len 0 --ind 1
-osmo-auc-gen (C) 2011-2012 by Harald Welte
+osmo-auc-gen (C) 2011-2023 by Harald Welte
 This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY
 
diff --git a/utils/osmo-auc-gen.c b/utils/osmo-auc-gen.c
index 86abeda..50419a4 100644
--- a/utils/osmo-auc-gen.c
+++ b/utils/osmo-auc-gen.c
@@ -98,6 +98,7 @@
 		"-s  --sqn\tSpecify SQN (only for 3G)\n"
 		"-i  --ind\tSpecify IND slot for new SQN after AUTS (only for 3G)\n"
 		"-l  --ind-len\tSpecify IND bit length (default=5) (only for 3G)\n"
+		"-L  --res-len\tSpecify RES byte length (default=8) (only for 3G)\n"
 		"-A  --auts\tSpecify AUTS (only for 3G)\n"
 		"-r  --rand\tSpecify random value\n"
 		"-I  --ipsec\tOutput in triplets.dat format for strongswan\n");
@@ -123,10 +124,12 @@
 	int fmt_triplets_dat = 0;
 	uint64_t ind_mask = 0;
 
-	printf("osmo-auc-gen (C) 2011-2012 by Harald Welte\n");
+	printf("osmo-auc-gen (C) 2011-2023 by Harald Welte\n");
 	printf("This is FREE SOFTWARE with ABSOLUTELY NO WARRANTY\n\n");
 
 	memset(_auts, 0, sizeof(_auts));
+	memset(vec, 0, sizeof(*vec));
+	vec->res_len = 8; /* default */
 
 	while (1) {
 		int c;
@@ -141,6 +144,7 @@
 			{ "sqn", 1, 0, 's' },
 			{ "ind", 1, 0, 'i' },
 			{ "ind-len", 1, 0, 'l' },
+			{ "res-len", 1, 0, 'L' },
 			{ "rand", 1, 0, 'r' },
 			{ "auts", 1, 0, 'A' },
 			{ "help", 0, 0, 'h' },
@@ -149,7 +153,7 @@
 
 		rc = 0;
 
-		c = getopt_long(argc, argv, "23a:k:o:f:s:i:l:r:hO:A:I", long_options,
+		c = getopt_long(argc, argv, "23a:k:o:f:s:i:l:L:r:hO:A:I", long_options,
 				&option_index);
 
 		if (c == -1)
@@ -265,6 +269,14 @@
 			}
 			test_aud.u.umts.ind_bitlen = atoi(optarg);
 			break;
+		case 'L':
+			rc = atoi(optarg);
+			if (rc != 4 && rc != 8 && rc != 16) {
+				fprintf(stderr, "Invalid RES length %u\n", rc);
+				exit(2);
+			}
+			vec->res_len = rc;
+			break;
 		case 'r':
 			rc = osmo_hexparse(optarg, _rand, sizeof(_rand));
 			if (rc != sizeof(_rand)) {
@@ -313,8 +325,6 @@
 		exit(2);
 	}
 
-	memset(vec, 0, sizeof(*vec));
-
 	if (test_aud.type == OSMO_AUTH_TYPE_UMTS) {
 		uint64_t seq_1 = 1LL << test_aud.u.umts.ind_bitlen;
 		ind_mask = seq_1 - 1;