iprop: Support hierarchical iprop

lib/kadm5/iprop.8

@@ -139,6 +139,12 @@ There is a keep-alive feature logged in the master's
 file (e.g.\&
 .Pa /var/heimdal/slave-stats ) .
 .Pp
+Note that hierarchical replication is supported by running
+an
+.Nm ipropd-master
+on the same KDC as an
+.Nm ipropd-slave .
+.Pp
 Supported options for
 .Nm ipropd-master :
 .Bl -tag -width Ds

lib/kadm5/ipropd_master.c

@@ -47,6 +47,7 @@ static int time_before_missing;
 static int time_before_gone;
 
 const char *master_hostname;
+const char *pidfile_basename;
 
 static krb5_socket_t
 make_signal_socket (krb5_context context)
@@ -937,9 +938,13 @@ get_first(kadm5_server_context *server_context, int log_fd,
 
     ret = kadm5_log_get_version_fd(server_context, log_fd, LOG_VERSION_FIRST,
                                    initial_verp, initial_timep);
+    if (ret == HEIM_ERR_EOF)
+        ret = kadm5_log_get_version_fd(server_context, log_fd,
+                                       LOG_VERSION_UBER, initial_verp,
+                                       initial_timep);
     if (ret != 0) {
         flock(log_fd, LOCK_UN);
-        krb5_warnx(context, "could not read initial log entry");
+        krb5_warn(context, ret, "could not read initial log entry");
         return -1;
     }
 
@@ -1529,8 +1534,10 @@ static struct getargs args[] = {
       "port ipropd will listen to", "port"},
     { "detach", 0, arg_flag, &detach_from_console,
       "detach from console", NULL },
-    { "daemon-child",       0 ,      arg_integer, &daemon_child,
+    { "daemon-child", 0, arg_integer, &daemon_child,
       "private argument, do not use", NULL },
+    { "pidfile-basename", 0, arg_string, &pidfile_basename,
+      "basename of pidfile; private argument for testing", "NAME" },
     { "hostname", 0, arg_string, rk_UNCONST(&master_hostname),
       "hostname of master (if not same as hostname)", "hostname" },
     { "verbose", 0, arg_flag, &verbose, NULL, NULL },
@@ -1573,7 +1580,7 @@ main(int argc, char **argv)
 
     if (detach_from_console && daemon_child == -1)
         daemon_child = roken_detach_prep(argc, argv, "--daemon-child");
-    rk_pidfile(NULL);
+    rk_pidfile(pidfile_basename);
 
     ret = krb5_init_context(&context);
     if (ret)

lib/kadm5/ipropd_slave.c

@@ -43,7 +43,8 @@ static krb5_log_facility *log_facility;
 static char five_min[] = "5 min";
 static char *server_time_lost = five_min;
 static int time_before_lost;
-const char *slave_str = NULL;
+static const char *slave_str;
+static const char *pidfile_basename;
 
 static int
 connect_to_master (krb5_context context, const char *master,
@@ -305,9 +306,9 @@ append_to_log_file(krb5_context context,
 }
 
 static int
-receive_loop (krb5_context context,
+receive_loop(krb5_context context,
-	      krb5_storage *sp,
+	     krb5_storage *sp,
-	      kadm5_server_context *server_context)
+	     kadm5_server_context *server_context)
 {
     int ret;
     off_t left, right, off;
@@ -316,6 +317,11 @@ receive_loop (krb5_context context,
     if (verbose)
         krb5_warnx(context, "receiving diffs");
 
+    ret = kadm5_log_exclusivelock(server_context);
+    if (ret)
+        krb5_err(context, IPROPD_RESTART, ret,
+                 "Failed to lock iprop log for writes");
+
     /*
      * Seek to the first entry in the message from the master that is
      * past the current version of the local database.
@@ -433,6 +439,12 @@ receive(krb5_context context,
     if (ret)
         krb5_err(context, IPROPD_RESTART_SLOW, ret, "db->close");
 
+    (void) kadm5_log_sharedlock(server_context);
+    if (verbose)
+        krb5_warnx(context, "downgraded iprop log lock to shared");
+    kadm5_log_signal_master(server_context);
+    if (verbose)
+        krb5_warnx(context, "signaled master for hierarchical iprop");
     return ret2;
 }
 
@@ -481,6 +493,9 @@ reinit_log(krb5_context context,
     ret = kadm5_log_reinit(server_context, vno);
     if (ret)
         krb5_err(context, IPROPD_RESTART_SLOW, ret, "kadm5_log_reinit");
+    (void) kadm5_log_sharedlock(server_context);
+    if (verbose)
+        krb5_warnx(context, "downgraded iprop log lock to shared");
 }
 
 
@@ -500,6 +515,10 @@ receive_everything(krb5_context context, int fd,
 
     krb5_warnx(context, "receive complete database");
 
+    ret = kadm5_log_exclusivelock(server_context);
+    if (ret)
+        krb5_err(context, IPROPD_RESTART, ret,
+                 "Failed to lock iprop log for writes");
     ret = asprintf(&dbname, "%s-NEW", server_context->db->hdb_name);
     if (ret == -1)
         krb5_err(context, IPROPD_RESTART, ENOMEM, "asprintf");
@@ -673,8 +692,10 @@ static struct getargs args[] = {
       "port ipropd-slave will connect to", "port"},
     { "detach", 0, arg_flag, &detach_from_console,
       "detach from console", NULL },
-    { "daemon-child",       0 ,      arg_integer, &daemon_child,
+    { "daemon-child", 0, arg_integer, &daemon_child,
       "private argument, do not use", NULL },
+    { "pidfile-basename", 0, arg_string, &pidfile_basename,
+      "basename of pidfile; private argument for testing", "NAME" },
     { "hostname", 0, arg_string, rk_UNCONST(&slave_str),
       "hostname of slave (if not same as hostname)", "hostname" },
     { "verbose", 0, arg_flag, &verbose, NULL, NULL },
@@ -729,7 +750,7 @@ main(int argc, char **argv)
 
     if (detach_from_console && daemon_child == -1)
         daemon_child = roken_detach_prep(argc, argv, "--daemon-child");
-    rk_pidfile(NULL);
+    rk_pidfile(pidfile_basename);
 
     ret = krb5_init_context(&context);
     if (ret)
@@ -804,13 +825,16 @@ main(int argc, char **argv)
     if (ret)
 	krb5_err (context, 1, ret, "db->open");
 
-    ret = kadm5_log_init (server_context);
+    ret = kadm5_log_init(server_context);
     if (ret)
-	krb5_err (context, 1, ret, "kadm5_log_init");
+	krb5_err(context, 1, ret, "kadm5_log_init");
+    (void) kadm5_log_sharedlock(server_context);
+    if (verbose)
+        krb5_warnx(context, "downgraded iprop log lock to shared");
 
-    ret = server_context->db->hdb_close (context, server_context->db);
+    ret = server_context->db->hdb_close(context, server_context->db);
     if (ret)
-	krb5_err (context, 1, ret, "db->close");
+	krb5_err(context, 1, ret, "db->close");
 
     get_creds(context, keytab_str, &ccache, master);
 
@@ -977,18 +1001,30 @@ main(int argc, char **argv)
                 continue;
             }
 
+            /*
+             * It's unclear why we open th HDB and call kadm5_log_init() here.
+             *
+             * We don't need it to process the log entries we receive in the
+             * FOR_YOU case: we already call kadm5_log_recover() in receive() /
+             * receive_loop().  Maybe it's just just in case, though at the
+             * cost of synchronization with ipropd-master if we're running one
+             * for hierarchical iprop.
+             */
 	    ret = server_context->db->hdb_open(context,
 					       server_context->db,
 					       O_RDWR | O_CREAT, 0600);
 	    if (ret)
 		krb5_err (context, 1, ret, "db->open while handling a "
 			  "message from the master");
-
             ret = kadm5_log_init(server_context);
             if (ret) {
                 krb5_err(context, IPROPD_RESTART, ret, "kadm5_log_init while "
                          "handling a message from the master");
             }
+            (void) kadm5_log_sharedlock(server_context);
+            if (verbose)
+                krb5_warnx(context, "downgraded iprop log lock to shared");
+
 	    ret = server_context->db->hdb_close (context, server_context->db);
 	    if (ret)
 		krb5_err (context, 1, ret, "db->close while handling a "
@@ -1019,6 +1055,7 @@ main(int argc, char **argv)
                     krb5_warnx(context, "master sent us a full dump");
 		ret = receive_everything(context, master_fd, server_context,
 					 auth_context);
+                (void) kadm5_log_sharedlock(server_context);
                 if (ret == 0) {
                     ret = ihave(context, auth_context, master_fd,
                                 server_context->log_context.version);
@@ -1027,6 +1064,11 @@ main(int argc, char **argv)
 		    connected = FALSE;
                 else
                     is_up_to_date(context, status_file, server_context);
+                if (verbose)
+                    krb5_warnx(context, "downgraded iprop log lock to shared");
+                kadm5_log_signal_master(server_context);
+                if (verbose)
+                    krb5_warnx(context, "signaled master for hierarchical iprop");
 		break;
 	    case ARE_YOU_THERE :
                 if (verbose)

lib/kadm5/libkadm5srv-exports.def

@@ -61,7 +61,9 @@ EXPORTS
 	kadm5_s_init_with_creds
 	kadm5_s_chpass_principal_cond
 	kadm5_s_create_principal_with_key
+	kadm5_log_exclusivelock
 	kadm5_log_set_version
+	kadm5_log_sharedlock
 	kadm5_log_signal_master
 ;!	kadm5_log_signal_socket
 	kadm5_log_signal_socket_info    ;!

lib/kadm5/log.c

@@ -682,6 +682,40 @@ kadm5_log_init(kadm5_server_context *server_context)
     return log_init(server_context, LOCK_EX);
 }
 
+/* Upgrade log lock to exclusive */
+kadm5_ret_t
+kadm5_log_exclusivelock(kadm5_server_context *server_context)
+{
+    kadm5_log_context *log_context = &server_context->log_context;
+
+    if (log_context->lock_mode == LOCK_EX)
+        return 0;
+    if (log_context->log_fd == -1)
+        return EINVAL;
+    if (flock(log_context->log_fd, LOCK_EX) < 0)
+        return errno;
+    log_context->read_only = 0;
+    log_context->lock_mode = LOCK_EX;
+    return 0;
+}
+
+/* Downgrade log lock to shared */
+kadm5_ret_t
+kadm5_log_sharedlock(kadm5_server_context *server_context)
+{
+    kadm5_log_context *log_context = &server_context->log_context;
+
+    if (log_context->lock_mode == LOCK_SH)
+        return 0;
+    if (log_context->log_fd == -1)
+        return EINVAL;
+    if (flock(log_context->log_fd, LOCK_SH) < 0)
+        return errno;
+    log_context->read_only = 1;
+    log_context->lock_mode = LOCK_SH;
+    return 0;
+}
+
 /* Open the log with an exclusive non-blocking lock */
 kadm5_ret_t
 kadm5_log_init_nb(kadm5_server_context *server_context)

lib/kadm5/version-script.map

@@ -64,7 +64,9 @@ HEIMDAL_KAMD5_SERVER_1.0 {
 		kadm5_s_init_with_creds;
 		kadm5_s_chpass_principal_cond;
 		kadm5_s_create_principal_with_key;
+		kadm5_log_exclusivelock;
 		kadm5_log_set_version;
+		kadm5_log_sharedlock;
 		kadm5_log_signal_master;
 		kadm5_log_signal_socket;
 		kadm5_log_previous;

tests/kdc/Makefile.am

@@ -15,6 +15,7 @@ noinst_DATA = \
 	krb5-bx509.conf \
 	krb5-httpkadmind.conf \
 	krb5-pkinit-win.conf \
+	krb5-master2.conf \
 	krb5-slave2.conf \
 	krb5-slave.conf
 
@@ -51,6 +52,7 @@ pwport = 49191
 restport = 49192
 restport2 = 49193
 ipropport = 49194
+ipropport2 = 49195
 
 if HAVE_DLOPEN
 do_dlopen = -e 's,[@]DLOPEN[@],true,g'
@@ -70,6 +72,7 @@ do_subst = $(heim_verbose)sed $(do_dlopen) \
 	-e 's,[@]restport2[@],$(restport2),g' \
 	-e 's,[@]pwport[@],$(pwport),g' \
 	-e 's,[@]ipropport[@],$(ipropport),g' \
+	-e 's,[@]ipropport2[@],$(ipropport2),g' \
 	-e 's,[@]objdir[@],$(top_builddir)/tests/kdc,g' \
 	-e 's,[@]top_builddir[@],$(top_builddir),g' \
 	-e 's,[@]db_type[@],$(db_type),g' \
@@ -165,7 +168,7 @@ check-httpkadmind: check-httpkadmind.in Makefile krb5-httpkadmind.conf
 	$(chmod) +x check-httpkadmind.tmp && \
 	mv check-httpkadmind.tmp check-httpkadmind
 
-check-iprop: check-iprop.in Makefile krb5.conf krb5-slave.conf krb5-slave2.conf
+check-iprop: check-iprop.in Makefile krb5.conf krb5-master2.conf krb5-slave.conf krb5-slave2.conf
 	$(do_subst) < $(srcdir)/check-iprop.in > check-iprop.tmp && \
 	$(chmod) +x check-iprop.tmp && \
 	mv check-iprop.tmp check-iprop
@@ -193,6 +196,9 @@ krb5.conf: krb5.conf.in Makefile
 	$(do_subst) \
 	   -e 's,[@]WEAK[@],false,g' \
 	   -e 's,[@]dk[@],,g' \
+	   -e 's,[@]messages[@],messages,g' \
+	   -e 's,[@]ipropstats[@],iprop-stats,g' \
+	   -e 's,[@]signalsocket[@],signal,g' \
 	   -e 's,[@]kdc[@],,g' < $(srcdir)/krb5.conf.in > krb5.conf.tmp && \
 	mv krb5.conf.tmp krb5.conf
 
@@ -236,6 +242,9 @@ krb5-weak.conf: krb5.conf.in Makefile
 	$(do_subst) \
 	   -e 's,[@]WEAK[@],true,g' \
 	   -e 's,[@]dk[@],default_keys = aes256-cts-hmac-sha1-96:pw-salt arcfour-hmac-md5:pw-salt des3-cbc-sha1:pw-salt des:pw-salt,g' \
+	   -e 's,[@]messages[@],messages,g' \
+	   -e 's,[@]signalsocket[@],signal,g' \
+	   -e 's,[@]ipropstats[@],iprop-stats,g' \
 	   -e 's,[@]kdc[@],,g' < $(srcdir)/krb5.conf.in > krb5-weak.conf.tmp && \
 	mv krb5-weak.conf.tmp krb5-weak.conf
 
@@ -243,13 +252,29 @@ krb5-slave.conf: krb5.conf.in Makefile
 	$(do_subst) \
 	   -e 's,[@]WEAK[@],true,g' \
 	   -e 's,[@]dk[@],,g' \
+	   -e 's,[@]messages[@],messages,g' \
+	   -e 's,[@]signalsocket[@],signal2,g' \
+	   -e 's,[@]ipropstats[@],iprop-stats,g' \
 	   -e 's,[@]kdc[@],.slave,g' < $(srcdir)/krb5.conf.in > krb5-slave.conf.tmp && \
 	mv krb5-slave.conf.tmp krb5-slave.conf
 
+krb5-master2.conf: krb5.conf.in Makefile
+	$(do_subst) \
+	   -e 's,[@]WEAK[@],true,g' \
+	   -e 's,[@]dk[@],,g' \
+	   -e 's,[@]messages[@],messages2,g' \
+	   -e 's,[@]signalsocket[@],signal2,g' \
+	   -e 's,[@]ipropstats[@],iprop-stats2,g' \
+	   -e 's,[@]kdc[@],.slave,g' < $(srcdir)/krb5.conf.in > krb5-master2.conf.tmp && \
+	mv krb5-master2.conf.tmp krb5-master2.conf
+
 krb5-slave2.conf: krb5.conf.in Makefile
 	$(do_subst) \
 	   -e 's,[@]WEAK[@],true,g' \
 	   -e 's,[@]dk[@],,g' \
+	   -e 's,[@]messages[@],messages2,g' \
+	   -e 's,[@]signalsocket[@],signal3,g' \
+	   -e 's,[@]ipropstats[@],iprop-stats2,g' \
 	   -e 's,[@]kdc[@],.slave2,g' < $(srcdir)/krb5.conf.in > krb5-slave2.conf.tmp && \
 	mv krb5-slave2.conf.tmp krb5-slave2.conf
 
@@ -295,6 +320,7 @@ CLEANFILES= \
 	foopassword \
 	foopassword.rkpty \
 	iprop-stats \
+	iprop-stats2 \
 	iprop.keytab \
 	ipropd.dumpfile \
 	kdc-tester4.json \
@@ -333,6 +359,8 @@ CLEANFILES= \
 	sdigest-reply \
 	server.keytab \
 	signal \
+	signal2 \
+	signal3 \
 	tempfile \
 	test-rc-file.rc \
 	ukt \

tests/kdc/check-iprop.in

@@ -50,6 +50,7 @@ R=TEST.H5L.SE
 
 port=@port@
 ipropport=@ipropport@
+ipropport2=@ipropport2@
 
 cache="FILE:${objdir}/cache.krb5"
 keytabfile=${objdir}/iprop.keytab
@@ -59,6 +60,60 @@ kdc="${kdc} --addresses=localhost -P $port"
 kadmin="${kadmin} -r $R"
 kinit="${kinit} -c $cache ${afs_no_afslog}"
 
+# We'll test iprop, and in particular, hierarchical iprop.  This means we'll
+# have a setup like:
+#
+#  ipropd-master -> ipropd-slave -> 2nd ipropd-master -> 2nd ipropd-slave
+
+# Waiting for incremental propagation is inherently difficult because we don't
+# have a way for ipropd-slave to signal this script that it has received
+# updates.  Well, it does have a way to signal a possible ipropd-master for
+# hierarchical iprop, but we don't have a way to get that signal here.
+#
+# FIXME: Add a private interface for async waiting for iprop.
+#
+# What we do is we have a set of utility functions:
+#
+#  - get_iprop_ver   [N] -> checks that N (default to 1) ops have made it over
+#  - get_iprop_ver2  [N] -> same, but for second ipropd-slave instance
+#
+#  - wait_for            -> repeat a command until it succeeds or too many tries
+#  - wait_for_slave  [N] -> wait for N ops to make it over (calls get_iprop_ver)
+#  - wait_for_slave2 [N] -> same, but for second ipropd-slave instance
+#
+# In particular the wait_for* functions busy-wait for a max amount of time,
+# with sleeps in between.
+#
+# NOTE: get_iprop_ver and get_iprop_ver2 keep hidden state.
+#
+# When first called, get_iprop_ver / get_iprop_ver2 save the current version
+# numbers.  Thereafter they check that N ops have been received.
+#
+# It is critical to account for every incremental op via get_iprop_ver /
+# get_iprop_ver2, or wait_for_slave / wait_for_slave2, otherwise this test will
+# be racy and will have spurious failures!
+#
+# The pattern should be something like this:
+#
+#   echo "Add host"
+#   ${kadmin} -l add --random-key --use-defaults host/foo@${R} || exit 1
+#   wait_for_slave
+#   ^^^^^^^^^^^^^^
+#      waits for 1 operation
+#
+# or
+#
+#   echo "Rollover host keys"
+#   ${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1
+#   ${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1
+#   ${kadmin} -l cpw -r --keepold host/foo@${R} || exit 1
+#   wait_for_slave 3
+#   ^^^^^^^^^^^^^^^^
+#      waits for the three operations
+#
+# So though all operations must be accounted for, they need not be accounted
+# one by one.
+
 slave_ver_from_master_old=
 slave_ver_from_master_new=
 slave_ver_old=
@@ -84,6 +139,31 @@ get_iprop_ver () {
     return 0
 }
 
+slave_ver_from_master_old2=
+slave_ver_from_master_new2=
+slave_ver_old2=
+slave_ver_new2=
+get_iprop_ver2 () {
+    min_change=${1:-1}
+    slave_ver_from_master_new2=`grep '^iprop/' iprop-stats2 | head -1 | awk '{print $3}'`
+    slave_ver_new2=`grep 'up-to-date with version:' iprop-slave-status2 | awk '{print $4}'`
+    if [ -z "$slave_ver_from_master_new2" -o -z "$slave_ver_new2" ]; then
+        return 1
+    fi
+    if [ x"$slave_ver_from_master_new2" != x"$slave_ver_new2" ]; then
+        return 1
+    fi
+    if [ x"$slave_ver_from_master_old2" != x ]; then
+        change=`expr "$slave_ver_from_master_new2" - "$slave_ver_from_master_old2"`
+        if [ "$change" -lt "$min_change" ]; then
+            return 1
+        fi
+    fi
+    slave_ver_from_master_old2=$slave_ver_from_master_new2
+    slave_ver_old2=$slave_ver_new2
+    return 0
+}
+
 waitsec=65
 sleeptime=2
 wait_for () {
@@ -119,6 +199,10 @@ wait_for_slave () {
     wait_for "iprop versions to change and/or slave to catch up" get_iprop_ver "$@"
 }
 
+wait_for_slave2 () {
+    wait_for "iprop versions to change and/or second slave to catch up" get_iprop_ver2 "$@"
+}
+
 wait_for_master_down () {
     wait_for "master to exit" check_pidfile_is_dead ipropd-master
 }
@@ -135,9 +219,10 @@ rm -f current-db*
 rm -f current*.log
 rm -f out-*
 rm -f mkey.file*
-rm -f messages.log
+rm -f messages.log messages.log
 
 > messages.log
+> messages.log2
 
 echo Creating database
 ${kadmin} -l \
@@ -194,8 +279,11 @@ ipdm=
 kdcpid=
 
 > iprop-stats
-rm -f iprop-slave-status
+> iprop-stats2
+rm -f iprop-slave-status iprop-slave-status2
 
+ipropd_slave2=$ipropd_slave
+ipropd_master2=$ipropd_master
 ipropd_slave="${ipropd_slave} --status-file=iprop-slave-status --port=$ipropport"
 ipropd_slave="${ipropd_slave} --hostname=slave.test.h5l.se -k ${keytab}"
 ipropd_slave="${ipropd_slave} --detach localhost"
@@ -203,7 +291,27 @@ ipropd_master="${ipropd_master} --hostname=localhost -k ${keytab}"
 ipropd_master="${ipropd_master} --port=$ipropport"
 ipropd_master="${ipropd_master} --database=${objdir}/current-db --detach"
 
-trap "echo 'killing ipropd s + m + kdc'; kill -9 \${ipdm} \${ipds} \${kdcpid} >/dev/null 2>/dev/null; tail messages.log ; tail iprop-stats; exit 1;" EXIT
+ipropd_slave2="${ipropd_slave2} --status-file=iprop-slave-status2 --port=$ipropport2"
+ipropd_slave2="${ipropd_slave2} --hostname=slave.test.h5l.se -k ${keytab}"
+ipropd_slave2="${ipropd_slave2} --pidfile-basename=ipropd-slave2"
+ipropd_slave2="${ipropd_slave2} --detach localhost"
+ipropd_master2="${ipropd_master2} --hostname=localhost -k ${keytab}"
+ipropd_master2="${ipropd_master2} --port=$ipropport2"
+ipropd_master2="${ipropd_master2} --pidfile-basename=ipropd-master2"
+ipropd_master2="${ipropd_master2} --database=${objdir}/current-db.slave --detach"
+
+cleanup() {
+    echo 'killing ipropd s + m + kdc'
+    test -n "$ipdm" && kill -9 $ipdm        >/dev/null 2>/dev/null
+    test -n "$ipdm2" && kill -9 $ipdm2      >/dev/null 2>/dev/null
+    test -n "$ipds" && kill -9 $ipds        >/dev/null 2>/dev/null
+    test -n "$ipds2" && kill -9 $ipds2      >/dev/null 2>/dev/null
+    test -n "$kdcpid" && kill -9 $kdcpid    >/dev/null 2>/dev/null
+    tail messages.log
+    tail iprop-stats
+    exit 1
+}
+trap cleanup EXIT
 
 echo Starting kdc ; > messages.log
 ${kdc} --detach --testing || { echo "kdc failed to start"; exit 1; }
@@ -226,6 +334,22 @@ echo "checking slave is up"
 ${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null || exit 1
 ${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave to up to date" ; cat iprop-slave-status ; exit 1; }
 
+# Also setup a second master on the slave, then a second slave to pull from the
+# second master.
+echo "starting master2" ; > messages.log
+env ${HEIM_MALLOC_DEBUG} \
+KRB5_CONFIG="${objdir}/krb5-master2.conf" \
+${ipropd_master2} || { echo "second ipropd-master failed to start"; exit 1; }
+ipdm2=`getpid ipropd-master2`
+
+echo "starting slave2" ; > messages.log
+env ${HEIM_MALLOC_DEBUG} \
+KRB5_CONFIG="${objdir}/krb5-slave2.conf" \
+${ipropd_slave2} || { echo "ipropd-slave failed to start"; exit 1; }
+ipds2=`getpid ipropd-slave2`
+sh ${wait_kdc} ipropd-slave messages2.log 'slave status change: up-to-date' || exit 1
+wait_for "Slave sees new host" get_iprop_ver2 0 || exit 1
+
 # ----------------- checking: pushing lives changes
 
 slave_get() { KRB5_CONFIG="${objdir}/krb5-slave.conf" ${kadmin} -l get "$@"; }
@@ -242,6 +366,7 @@ slave_check_exists() {
 
 echo "Add host"
 ${kadmin} -l add --random-key --use-defaults host/foo@${R} || exit 1
+wait_for_slave
 wait_for "Slave sees new host" slave_check_exists "host/foo@${R}"
 
 echo "Rollover host keys"
@@ -254,6 +379,8 @@ slave_get host/foo@${R} | \
 ' | sed 's/^.*[[]\(.*\)[]].*$/\1/' | grep '[0-9]' | sort -nu | tr -d '
 ' | ${EGREP} 1234 > /dev/null || exit 1
 
+wait_for_slave2 4
+
 echo "Delete 3DES keys"
 ${kadmin} -l del_enctype host/foo@${R} des3-cbc-sha1
 wait_for_slave
@@ -280,6 +407,8 @@ ${kadmin} -l get host/foo@${R} > /dev/null 2>/dev/null && exit 1
 KRB5_CONFIG="${objdir}/krb5-slave.conf" \
 ${kadmin} -l get host/bar@${R} > /dev/null || exit 1
 
+wait_for_slave2 3
+
 echo "Delete host"
 ${kadmin} -l delete host/bar@${R} || exit 1
 wait_for_slave
@@ -290,8 +419,11 @@ ${kadmin} -l get host/bar@${R} > /dev/null 2>/dev/null && exit 1
 echo "Re-add host"
 ${kadmin} -l add --random-key --use-defaults host/foo@${R} || exit 1
 ${kadmin} -l add --random-key --use-defaults host/bar@${R} || exit 1
+wait_for_slave 2
 wait_for "Slave sees re-added host" slave_check_exists "host/bar@${R}"
 
+wait_for_slave2 3
+
 echo "kill slave and remove log and database"
 > iprop-stats
 sh ${leaks_kill} ipropd-slave $ipds || exit 1
@@ -324,6 +456,7 @@ echo "checking slave is up again"
 wait_for "slave to start and connect to master" \
     ${EGREP} 'iprop/slave.test.h5l.se@TEST.H5L.SE.*Up' iprop-stats >/dev/null
 wait_for_slave 2
+wait_for_slave2 2
 ${EGREP} 'up-to-date with version' iprop-slave-status >/dev/null || { echo "slave not up to date" ; cat iprop-slave-status ; exit 1; }
 echo "checking for replay problems"
 ${EGREP} 'Entry already exists in database' messages.log && exit 1
@@ -359,6 +492,7 @@ ${EGREP} 'Entry already exists in database' messages.log && exit 1
 
 ${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1
 wait_for_slave
+wait_for_slave2
 
 echo "live truncate on master log"
 ${iprop_log} truncate -K 5 || exit 1
@@ -408,6 +542,7 @@ ${EGREP} 'Entry already exists in database' messages.log && exit 1
 echo "pushing one change"
 ${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1
 wait_for_slave
+wait_for_slave2 0
 
 echo "Killing master"
 sh ${leaks_kill} ipropd-master $ipdm || exit 1
@@ -439,6 +574,7 @@ kill -0 ${ipds}  || { echo "slave no longer there"; exit 1; }
 echo "pushing one change"
 ${kadmin} -l cpw --random-password user@${R} > /dev/null || exit 1
 wait_for_slave
+wait_for_slave2
 
 echo "shutting down all services"
 
@@ -446,6 +582,8 @@ leaked=false
 sh ${leaks_kill} kdc $kdcpid || leaked=true
 sh ${leaks_kill} ipropd-master $ipdm || leaked=true
 sh ${leaks_kill} ipropd-slave $ipds || leaked=true
+sh ${leaks_kill} ipropd-master $ipdm2 || leaked=true
+sh ${leaks_kill} ipropd-slave $ipds2 || leaked=true
 rm -f iprop-slave-status
 trap "" EXIT
 $leaked && exit 1

tests/kdc/krb5.conf.in

@@ -114,8 +114,8 @@
 		}
 	}
 
-	signal_socket = @objdir@/signal
+	signal_socket = @objdir@/@signalsocket@
-	iprop-stats = @objdir@/iprop-stats
+	iprop-stats = @objdir@/@ipropstats@
 	iprop-acl = @srcdir@/iprop-acl
         log-max-size = 40000
 
@@ -123,14 +123,14 @@
 	db-dir = @objdir@
 
 [logging]
-	kdc = 0-/FILE:@objdir@/messages.log
+	kdc = 0-/FILE:@objdir@/@messages@.log
-	krb5 = 0-/FILE:@objdir@/messages.log
+	krb5 = 0-/FILE:@objdir@/@messages@.log
-	default = 0-/FILE:@objdir@/messages.log
+	default = 0-/FILE:@objdir@/@messages@.log
 
 # If you are doing preformance measurements on OSX you want to change
 # the kdc LOG line from = to - below to keep the FILE open and avoid
 # open/write/close which is blocking (rdar:// ) on OSX.
-#	kdc = 0-/FILE=@objdir@/messages.log
+#	kdc = 0-/FILE=@objdir@/@messages@.log
 
 [kadmin]
 	save-password = true