fix some 64-bit bugs in RPCSEC_GSS

	* auth_gss.h: Add prototype for xdr_rpc_gss_buf.

	* authgss_prot.c (xdr_rpc_gss_buf): New function.
	(xdr_rpc_gss_cred, xdr_rpc_gss_init_args, xdr_rpc_gss_init_res):
	(xdr_rpc_gss_wrap_data, xdr_rpc_gss_unwrap_data): Use
	xdr_rpc_gss_buf, to avoid passing a (size_t *) to a xdr_bytes(),
	which expects (u_int *).
	(xdr_rpc_gss_wrap_data): Use tmplen to avoid passing (size_t *) to
	xdr_u_int.

	* libgssrpc.exports:
	* rename.h: Add xdr_rpc_gss_buf.

	* xdr.c (xdr_enum): Fix (kind of) for 64-bit.  Still inherently
	bogus, due to casting of (enum foo *) to (enum_t *) by callers.

ticket: new
component: krb5-libs

git-svn-id: svn://anonsvn.mit.edu/krb5/trunk@16499 dc483132-0cff-0310-8789-dd5450dbe970

6 files changed, 63 insertions, 23 deletions

diff --git a/src/lib/rpc/ChangeLog b/src/lib/rpc/ChangeLog
index 3a246ac..81f61cf 100644
--- a/src/lib/rpc/ChangeLog
+++ b/src/lib/rpc/ChangeLog
@@ -1,3 +1,21 @@
+2004-06-18  Tom Yu  <tlyu@mit.edu>
+
+	* auth_gss.h: Add prototype for xdr_rpc_gss_buf.
+
+	* authgss_prot.c (xdr_rpc_gss_buf): New function.
+	(xdr_rpc_gss_cred, xdr_rpc_gss_init_args, xdr_rpc_gss_init_res):
+	(xdr_rpc_gss_wrap_data, xdr_rpc_gss_unwrap_data): Use
+	xdr_rpc_gss_buf, to avoid passing a (size_t *) to a xdr_bytes(),
+	which expects (u_int *).
+	(xdr_rpc_gss_wrap_data): Use tmplen to avoid passing (size_t *) to
+	xdr_u_int.
+
+	* libgssrpc.exports:
+	* rename.h: Add xdr_rpc_gss_buf.
+
+	* xdr.c (xdr_enum): Fix (kind of) for 64-bit.  Still inherently
+	bogus, due to casting of (enum foo *) to (enum_t *) by callers.
+
 2004-06-17  Tom Yu  <tlyu@mit.edu>
 
 	* libgssrpc.exports: Update.
diff --git a/src/lib/rpc/auth_gss.h b/src/lib/rpc/auth_gss.h
index 0bf701d..1ea12cb 100644
--- a/src/lib/rpc/auth_gss.h
+++ b/src/lib/rpc/auth_gss.h
@@ -110,6 +110,7 @@ struct rpc_gss_init_res {
 #define MAXSEQ		0x80000000
 
 /* Prototypes. */
+bool_t	xdr_rpc_gss_buf		(XDR *xdrs, gss_buffer_t, u_int maxsize);
 bool_t	xdr_rpc_gss_cred	(XDR *xdrs, struct rpc_gss_cred *p);
 bool_t	xdr_rpc_gss_init_args	(XDR *xdrs, gss_buffer_desc *p);
 bool_t	xdr_rpc_gss_init_res	(XDR *xdrs, struct rpc_gss_init_res *p);
diff --git a/src/lib/rpc/authgss_prot.c b/src/lib/rpc/authgss_prot.c
index 6d91b8b..3224985 100644
--- a/src/lib/rpc/authgss_prot.c
+++ b/src/lib/rpc/authgss_prot.c
@@ -47,17 +47,36 @@
 #endif
 
 bool_t
+xdr_rpc_gss_buf(XDR *xdrs, gss_buffer_t buf, u_int maxsize)
+{
+	bool_t xdr_stat;
+	u_int tmplen;
+
+	if (xdrs->x_op == XDR_ENCODE) {
+		if (buf->length > UINT_MAX)
+			return (FALSE);
+		else
+			tmplen = buf->length;
+	}
+	xdr_stat = xdr_bytes(xdrs, &buf->value, &tmplen, maxsize);
+
+	if (xdr_stat && xdrs->x_op == XDR_DECODE)
+		buf->length = tmplen;
+
+	return (xdr_stat);
+}
+
+bool_t
 xdr_rpc_gss_cred(XDR *xdrs, struct rpc_gss_cred *p)
 {
 	bool_t xdr_stat;
-	
+
 	xdr_stat = (xdr_u_int(xdrs, &p->gc_v) &&
 		    xdr_enum(xdrs, (enum_t *)&p->gc_proc) &&
 		    xdr_u_int32(xdrs, &p->gc_seq) &&
 		    xdr_enum(xdrs, (enum_t *)&p->gc_svc) &&
-		    xdr_bytes(xdrs, (char **)&p->gc_ctx.value,
-			      &p->gc_ctx.length, MAX_AUTH_BYTES));
-	
+		    xdr_rpc_gss_buf(xdrs, &p->gc_ctx, MAX_AUTH_BYTES));
+
 	log_debug("xdr_rpc_gss_cred: %s %s "
 		  "(v %d, proc %d, seq %d, svc %d, ctx %p:%d)",
 		  (xdrs->x_op == XDR_ENCODE) ? "encode" : "decode",
@@ -72,9 +91,8 @@ bool_t
 xdr_rpc_gss_init_args(XDR *xdrs, gss_buffer_desc *p)
 {
 	bool_t xdr_stat;
-	
-	xdr_stat = xdr_bytes(xdrs, (char **)&p->value,
-			      &p->length, MAX_NETOBJ_SZ);
+
+	xdr_stat = xdr_rpc_gss_buf(xdrs, p, MAX_NETOBJ_SZ);
 
 	log_debug("xdr_rpc_gss_init_args: %s %s (token %p:%d)",
 		  (xdrs->x_op == XDR_ENCODE) ? "encode" : "decode",
@@ -88,14 +106,12 @@ bool_t
 xdr_rpc_gss_init_res(XDR *xdrs, struct rpc_gss_init_res *p)
 {
 	bool_t xdr_stat;
-	
-	xdr_stat = (xdr_bytes(xdrs, (char **)&p->gr_ctx.value,
-			      &p->gr_ctx.length, MAX_NETOBJ_SZ) &&
+
+	xdr_stat = (xdr_rpc_gss_buf(xdrs, &p->gr_ctx, MAX_NETOBJ_SZ) &&
 		    xdr_u_int32(xdrs, &p->gr_major) &&
 		    xdr_u_int32(xdrs, &p->gr_minor) &&
 		    xdr_u_int32(xdrs, &p->gr_win) &&
-		    xdr_bytes(xdrs, (char **)&p->gr_token.value,
-			      &p->gr_token.length, MAX_NETOBJ_SZ));
+		    xdr_rpc_gss_buf(xdrs, &p->gr_token, MAX_NETOBJ_SZ));
 
 	log_debug("xdr_rpc_gss_init_res %s %s "
 		  "(ctx %p:%d, maj %d, min %d, win %d, token %p:%d)",
@@ -117,6 +133,7 @@ xdr_rpc_gss_wrap_data(XDR *xdrs, xdrproc_t xdr_func, caddr_t xdr_ptr,
 	OM_uint32	maj_stat, min_stat;
 	int		start, end, conf_state;
 	bool_t		xdr_stat;
+	u_int		tmplen;
 
 	/* Skip databody length. */
 	start = XDR_GETPOS(xdrs);
@@ -137,7 +154,12 @@ xdr_rpc_gss_wrap_data(XDR *xdrs, xdrproc_t xdr_func, caddr_t xdr_ptr,
 	if (svc == RPCSEC_GSS_SVC_INTEGRITY) {
 		/* Marshal databody_integ length. */
 		XDR_SETPOS(xdrs, start);
-		if (!xdr_u_int(xdrs, &databuf.length))
+		if (databuf.length > UINT_MAX)
+			return (FALSE);
+		else
+			tmplen = databuf.length;
+
+		if (!xdr_u_int(xdrs, &tmplen))
 			return (FALSE);
 		
 		/* Checksum rpc_gss_data_t. */
@@ -149,8 +171,7 @@ xdr_rpc_gss_wrap_data(XDR *xdrs, xdrproc_t xdr_func, caddr_t xdr_ptr,
 		}
 		/* Marshal checksum. */
 		XDR_SETPOS(xdrs, end);
-		xdr_stat = xdr_bytes(xdrs, (char **)&wrapbuf.value,
-				     &wrapbuf.length, MAX_NETOBJ_SZ);
+		xdr_stat = xdr_rpc_gss_buf(xdrs, &wrapbuf, MAX_NETOBJ_SZ);
 		gss_release_buffer(&min_stat, &wrapbuf);
 	}		
 	else if (svc == RPCSEC_GSS_SVC_PRIVACY) {
@@ -163,8 +184,7 @@ xdr_rpc_gss_wrap_data(XDR *xdrs, xdrproc_t xdr_func, caddr_t xdr_ptr,
 		}
 		/* Marshal databody_priv. */
 		XDR_SETPOS(xdrs, start);
-		xdr_stat = xdr_bytes(xdrs, (char **)&wrapbuf.value,
-				     &wrapbuf.length, MAX_NETOBJ_SZ);
+		xdr_stat = xdr_rpc_gss_buf(xdrs, &wrapbuf, MAX_NETOBJ_SZ);
 		gss_release_buffer(&min_stat, &wrapbuf);
 	}
 	return (xdr_stat);
@@ -189,14 +209,12 @@ xdr_rpc_gss_unwrap_data(XDR *xdrs, xdrproc_t xdr_func, caddr_t xdr_ptr,
 	
 	if (svc == RPCSEC_GSS_SVC_INTEGRITY) {
 		/* Decode databody_integ. */
-		if (!xdr_bytes(xdrs, (char **)&databuf.value, &databuf.length,
-			       MAX_NETOBJ_SZ)) {
+		if (!xdr_rpc_gss_buf(xdrs, &databuf, MAX_NETOBJ_SZ)) {
 			log_debug("xdr decode databody_integ failed");
 			return (FALSE);
 		}
 		/* Decode checksum. */
-		if (!xdr_bytes(xdrs, (char **)&wrapbuf.value, &wrapbuf.length,
-			       MAX_NETOBJ_SZ)) {
+		if (!xdr_rpc_gss_buf(xdrs, &wrapbuf, MAX_NETOBJ_SZ)) {
 			gss_release_buffer(&min_stat, &databuf);
 			log_debug("xdr decode checksum failed");
 			return (FALSE);
@@ -214,8 +232,7 @@ xdr_rpc_gss_unwrap_data(XDR *xdrs, xdrproc_t xdr_func, caddr_t xdr_ptr,
 	}
 	else if (svc == RPCSEC_GSS_SVC_PRIVACY) {
 		/* Decode databody_priv. */
-		if (!xdr_bytes(xdrs, (char **)&wrapbuf.value, &wrapbuf.length,
-			       MAX_NETOBJ_SZ)) {
+		if (!xdr_rpc_gss_buf(xdrs, &wrapbuf, MAX_NETOBJ_SZ)) {
 			log_debug("xdr decode databody_priv failed");
 			return (FALSE);
 		}
diff --git a/src/lib/rpc/libgssrpc.exports b/src/lib/rpc/libgssrpc.exports
index 471014a..de3280f 100644
--- a/src/lib/rpc/libgssrpc.exports
+++ b/src/lib/rpc/libgssrpc.exports
@@ -117,6 +117,7 @@ gssrpc_xdr_rejected_reply
 gssrpc_xdr_replymsg
 gssrpc_xdr_rmtcall_args
 gssrpc_xdr_rmtcallres
+gssrpc_xdr_rpc_gss_buf
 gssrpc_xdr_rpc_gss_cred
 gssrpc_xdr_rpc_gss_data
 gssrpc_xdr_rpc_gss_init_args
diff --git a/src/lib/rpc/rename.h b/src/lib/rpc/rename.h
index 2cc4232..4f5971d 100644
--- a/src/lib/rpc/rename.h
+++ b/src/lib/rpc/rename.h
@@ -82,6 +82,7 @@
 
 /* auth_gss.h */
 
+#define xdr_rpc_gss_buf		gssrpc_xdr_rpc_gss_buf
 #define xdr_rpc_gss_cred	gssrpc_xdr_rpc_gss_cred
 #define xdr_rpc_gss_init_args	gssrpc_xdr_rpc_gss_init_args
 #define xdr_rpc_gss_init_res	gssrpc_xdr_rpc_gss_init_res
diff --git a/src/lib/rpc/xdr.c b/src/lib/rpc/xdr.c
index 2a9eae8..3bec451 100644
--- a/src/lib/rpc/xdr.c
+++ b/src/lib/rpc/xdr.c
@@ -347,6 +347,8 @@ xdr_enum(xdrs, ep)
 	 */
 	if (sizeof (enum sizecheck) == sizeof (long)) {
 		return (xdr_long(xdrs, (long *)ep));
+	} else if (sizeof (enum sizecheck) == sizeof (int)) {
+		return (xdr_int(xdrs, (int *)ep));
 	} else if (sizeof (enum sizecheck) == sizeof (short)) {
 		return (xdr_short(xdrs, (short *)ep));
 	} else {