multipathd: issue systemd READY after initial configuration
[multipath-tools/.git] / multipathd / main.c
index 7d6d9b5..3534afb 100644 (file)
 #endif
 #include <semaphore.h>
 #include <time.h>
+#include <stdbool.h>
+
+/*
+ * libmultipath
+ */
+#include "time-util.h"
 
 /*
  * libcheckers
@@ -79,6 +85,7 @@ int uxsock_timeout;
 #include "lock.h"
 #include "waiter.h"
 #include "wwids.h"
+#include "../third-party/valgrind/drd.h"
 
 #define FILE_NAME_SIZE 256
 #define CMDSIZE 160
@@ -106,7 +113,7 @@ int ignore_new_devs;
 enum daemon_status running_state = DAEMON_INIT;
 pid_t daemon_pid;
 pthread_mutex_t config_lock = PTHREAD_MUTEX_INITIALIZER;
-pthread_cond_t config_cond = PTHREAD_COND_INITIALIZER;
+pthread_cond_t config_cond;
 
 /*
  * global copy of vecs for use in sig handlers
@@ -193,7 +200,7 @@ int set_config_state(enum daemon_status state)
                if (running_state != DAEMON_IDLE) {
                        struct timespec ts;
 
-                       clock_gettime(CLOCK_REALTIME, &ts);
+                       clock_gettime(CLOCK_MONOTONIC, &ts);
                        ts.tv_sec += 1;
                        rc = pthread_cond_timedwait(&config_cond,
                                                    &config_lock, &ts);
@@ -451,7 +458,7 @@ uev_add_map (struct uevent * uev, struct vectors * vecs)
                }
        }
        pthread_cleanup_push(cleanup_lock, &vecs->lock);
-       lock(vecs->lock);
+       lock(&vecs->lock);
        pthread_testcancel();
        rc = ev_add_map(uev->kernel, alias, vecs);
        lock_cleanup_pop(vecs->lock);
@@ -479,6 +486,8 @@ ev_add_map (char * dev, char * alias, struct vectors * vecs)
 
        if (mpp) {
                if (mpp->wait_for_udev > 1) {
+                       condlog(2, "%s: performing delayed actions",
+                               mpp->alias);
                        if (update_map(mpp, vecs))
                                /* setup multipathd removed the map */
                                return 1;
@@ -557,7 +566,7 @@ uev_remove_map (struct uevent * uev, struct vectors * vecs)
        minor = uevent_get_minor(uev);
 
        pthread_cleanup_push(cleanup_lock, &vecs->lock);
-       lock(vecs->lock);
+       lock(&vecs->lock);
        pthread_testcancel();
        mpp = find_mp_by_minor(vecs->mpvec, minor);
 
@@ -602,7 +611,7 @@ ev_remove_map (char * devname, char * alias, int minor, struct vectors * vecs)
 }
 
 static int
-uev_add_path (struct uevent *uev, struct vectors * vecs)
+uev_add_path (struct uevent *uev, struct vectors * vecs, int need_do_map)
 {
        struct path *pp;
        int ret = 0, i;
@@ -618,7 +627,7 @@ uev_add_path (struct uevent *uev, struct vectors * vecs)
        }
 
        pthread_cleanup_push(cleanup_lock, &vecs->lock);
-       lock(vecs->lock);
+       lock(&vecs->lock);
        pthread_testcancel();
        pp = find_path_by_dev(vecs->pathvec, uev->kernel);
        if (pp) {
@@ -635,7 +644,7 @@ uev_add_path (struct uevent *uev, struct vectors * vecs)
                                     DI_ALL | DI_BLACKLIST);
                        put_multipath_config(conf);
                        if (r == PATHINFO_OK)
-                               ret = ev_add_path(pp, vecs);
+                               ret = ev_add_path(pp, vecs, need_do_map);
                        else if (r == PATHINFO_SKIPPED) {
                                condlog(3, "%s: remove blacklisted path",
                                        uev->kernel);
@@ -659,7 +668,7 @@ uev_add_path (struct uevent *uev, struct vectors * vecs)
         */
        conf = get_multipath_config();
        ret = alloc_path_with_pathinfo(conf, uev->udev,
-                                      DI_ALL, &pp);
+                                      uev->wwid, DI_ALL, &pp);
        put_multipath_config(conf);
        if (!pp) {
                if (ret == PATHINFO_SKIPPED)
@@ -668,14 +677,14 @@ uev_add_path (struct uevent *uev, struct vectors * vecs)
                return 1;
        }
        pthread_cleanup_push(cleanup_lock, &vecs->lock);
-       lock(vecs->lock);
+       lock(&vecs->lock);
        pthread_testcancel();
        ret = store_path(vecs->pathvec, pp);
        if (!ret) {
                conf = get_multipath_config();
                pp->checkint = conf->checkint;
                put_multipath_config(conf);
-               ret = ev_add_path(pp, vecs);
+               ret = ev_add_path(pp, vecs, need_do_map);
        } else {
                condlog(0, "%s: failed to store path info, "
                        "dropping event",
@@ -693,7 +702,7 @@ uev_add_path (struct uevent *uev, struct vectors * vecs)
  * 1: error
  */
 int
-ev_add_path (struct path * pp, struct vectors * vecs)
+ev_add_path (struct path * pp, struct vectors * vecs, int need_do_map)
 {
        struct multipath * mpp;
        char params[PARAMS_SIZE] = {0};
@@ -709,7 +718,11 @@ ev_add_path (struct path * pp, struct vectors * vecs)
                goto fail; /* leave path added to pathvec */
        }
        mpp = find_mp_by_wwid(vecs->mpvec, pp->wwid);
-       if (mpp && mpp->wait_for_udev) {
+       if (mpp && mpp->wait_for_udev &&
+           (pathcount(mpp, PATH_UP) > 0 ||
+            (pathcount(mpp, PATH_GHOST) > 0 && pp->tpgs != TPGS_IMPLICIT))) {
+               /* if wait_for_udev is set and valid paths exist */
+               condlog(2, "%s: delaying path addition until %s is fully initialized", pp->dev, mpp->alias);
                mpp->wait_for_udev = 2;
                orphan_path(pp, "waiting for create to complete");
                return 0;
@@ -758,6 +771,13 @@ rescan:
        /* persistent reservation check*/
        mpath_pr_event_handle(pp);
 
+       if (!need_do_map)
+               return 0;
+
+       if (!dm_map_present(mpp->alias)) {
+               mpp->action = ACT_CREATE;
+               start_waiter = 1;
+       }
        /*
         * push the map to the device-mapper
         */
@@ -824,18 +844,18 @@ fail:
 }
 
 static int
-uev_remove_path (struct uevent *uev, struct vectors * vecs)
+uev_remove_path (struct uevent *uev, struct vectors * vecs, int need_do_map)
 {
        struct path *pp;
        int ret;
 
        condlog(2, "%s: remove path (uevent)", uev->kernel);
        pthread_cleanup_push(cleanup_lock, &vecs->lock);
-       lock(vecs->lock);
+       lock(&vecs->lock);
        pthread_testcancel();
        pp = find_path_by_dev(vecs->pathvec, uev->kernel);
        if (pp)
-               ret = ev_remove_path(pp, vecs);
+               ret = ev_remove_path(pp, vecs, need_do_map);
        lock_cleanup_pop(vecs->lock);
        if (!pp) {
                /* Not an error; path might have been purged earlier */
@@ -846,7 +866,7 @@ uev_remove_path (struct uevent *uev, struct vectors * vecs)
 }
 
 int
-ev_remove_path (struct path *pp, struct vectors * vecs)
+ev_remove_path (struct path *pp, struct vectors * vecs, int need_do_map)
 {
        struct multipath * mpp;
        int i, retval = 0;
@@ -883,6 +903,7 @@ ev_remove_path (struct path *pp, struct vectors * vecs)
                                mpp->retry_tick = 0;
                                mpp->no_path_retry = NO_PATH_RETRY_FAIL;
                                mpp->flush_on_last_del = FLUSH_IN_PROGRESS;
+                               mpp->stat_map_failures++;
                                dm_queue_if_no_path(mpp->alias, 0);
                        }
                        if (!flush_map(mpp, vecs, 1)) {
@@ -908,6 +929,8 @@ ev_remove_path (struct path *pp, struct vectors * vecs)
                        goto out;
                }
 
+               if (!need_do_map)
+                       goto out;
                /*
                 * reload the map
                 */
@@ -947,50 +970,75 @@ static int
 uev_update_path (struct uevent *uev, struct vectors * vecs)
 {
        int ro, retval = 0;
+       struct path * pp;
+       struct config *conf;
+       int disable_changed_wwids;
+
+       conf = get_multipath_config();
+       disable_changed_wwids = conf->disable_changed_wwids;
+       put_multipath_config(conf);
 
        ro = uevent_get_disk_ro(uev);
 
-       if (ro >= 0) {
-               struct path * pp;
-               struct multipath *mpp = NULL;
+       pthread_cleanup_push(cleanup_lock, &vecs->lock);
+       lock(&vecs->lock);
+       pthread_testcancel();
 
-               condlog(2, "%s: update path write_protect to '%d' (uevent)",
-                       uev->kernel, ro);
-               pthread_cleanup_push(cleanup_lock, &vecs->lock);
-               lock(vecs->lock);
-               pthread_testcancel();
-               /*
-                * pthread_mutex_lock() and pthread_mutex_unlock()
-                * need to be at the same indentation level, hence
-                * this slightly convoluted codepath.
-                */
-               pp = find_path_by_dev(vecs->pathvec, uev->kernel);
-               if (pp) {
-                       if (pp->initialized == INIT_REQUESTED_UDEV) {
-                               retval = 2;
-                       } else {
-                               mpp = pp->mpp;
-                               if (mpp && mpp->wait_for_udev) {
-                                       mpp->wait_for_udev = 2;
-                                       mpp = NULL;
-                                       retval = 0;
+       pp = find_path_by_dev(vecs->pathvec, uev->kernel);
+       if (pp) {
+               struct multipath *mpp = pp->mpp;
+
+               if (disable_changed_wwids &&
+                   (strlen(pp->wwid) || pp->wwid_changed)) {
+                       char wwid[WWID_SIZE];
+
+                       strcpy(wwid, pp->wwid);
+                       get_uid(pp, pp->state, uev->udev);
+                       if (strcmp(wwid, pp->wwid) != 0) {
+                               condlog(0, "%s: path wwid changed from '%s' to '%s'. disallowing", uev->kernel, wwid, pp->wwid);
+                               strcpy(pp->wwid, wwid);
+                               if (!pp->wwid_changed) {
+                                       pp->wwid_changed = 1;
+                                       pp->tick = 1;
+                                       dm_fail_path(pp->mpp->alias, pp->dev_t);
                                }
-                       }
-                       if (mpp) {
-                               retval = reload_map(vecs, mpp, 0, 1);
+                               goto out;
+                       } else
+                               pp->wwid_changed = 0;
+               }
+
+               if (pp->initialized == INIT_REQUESTED_UDEV)
+                       retval = uev_add_path(uev, vecs, 1);
+               else if (mpp && ro >= 0) {
+                       condlog(2, "%s: update path write_protect to '%d' (uevent)", uev->kernel, ro);
 
+                       if (mpp->wait_for_udev)
+                               mpp->wait_for_udev = 2;
+                       else {
+                               retval = reload_map(vecs, mpp, 0, 1);
                                condlog(2, "%s: map %s reloaded (retval %d)",
                                        uev->kernel, mpp->alias, retval);
                        }
                }
-               lock_cleanup_pop(vecs->lock);
-               if (!pp) {
-                       condlog(0, "%s: spurious uevent, path not found",
-                               uev->kernel);
-                       return 1;
+       }
+out:
+       lock_cleanup_pop(vecs->lock);
+       if (!pp) {
+               /* If the path is blacklisted, print a debug/non-default verbosity message. */
+               if (uev->udev) {
+                       int flag = DI_SYSFS | DI_WWID;
+
+                       conf = get_multipath_config();
+                       retval = alloc_path_with_pathinfo(conf, uev->udev, uev->wwid, flag, NULL);
+                       put_multipath_config(conf);
+
+                       if (retval == PATHINFO_SKIPPED) {
+                               condlog(3, "%s: spurious uevent, path is blacklisted", uev->kernel);
+                               return 0;
+                       }
                }
-               if (retval == 2)
-                       return uev_add_path(uev, vecs);
+
+               condlog(0, "%s: spurious uevent, path not found", uev->kernel);
        }
 
        return retval;
@@ -1007,13 +1055,14 @@ map_discovery (struct vectors * vecs)
 
        vector_foreach_slot (vecs->mpvec, mpp, i)
                if (setup_multipath(vecs, mpp))
-                       return 1;
+                       i--;
 
        return 0;
 }
 
 int
-uxsock_trigger (char * str, char ** reply, int * len, void * trigger_data)
+uxsock_trigger (char * str, char ** reply, int * len, bool is_root,
+               void * trigger_data)
 {
        struct vectors * vecs;
        int r;
@@ -1022,7 +1071,15 @@ uxsock_trigger (char * str, char ** reply, int * len, void * trigger_data)
        *len = 0;
        vecs = (struct vectors *)trigger_data;
 
-       r = parse_cmd(str, reply, len, vecs, uxsock_timeout / 1000);
+       if ((str != NULL) && (is_root == false) &&
+           (strncmp(str, "list", strlen("list")) != 0) &&
+           (strncmp(str, "show", strlen("show")) != 0)) {
+               *reply = STRDUP("permission deny: need to be root");
+               *len = strlen(*reply) + 1;
+               return 1;
+       }
+
+       r = parse_cmd(str, reply, len, vecs, uxsock_timeout);
 
        if (r > 0) {
                if (r == ETIMEDOUT)
@@ -1042,40 +1099,15 @@ uxsock_trigger (char * str, char ** reply, int * len, void * trigger_data)
        return r;
 }
 
-static int
-uev_discard(char * devpath)
-{
-       char *tmp;
-       char a[11], b[11];
-
-       /*
-        * keep only block devices, discard partitions
-        */
-       tmp = strstr(devpath, "/block/");
-       if (tmp == NULL){
-               condlog(4, "no /block/ in '%s'", devpath);
-               return 1;
-       }
-       if (sscanf(tmp, "/block/%10s", a) != 1 ||
-           sscanf(tmp, "/block/%10[^/]/%10s", a, b) == 2) {
-               condlog(4, "discard event on %s", devpath);
-               return 1;
-       }
-       return 0;
-}
-
 int
 uev_trigger (struct uevent * uev, void * trigger_data)
 {
        int r = 0;
        struct vectors * vecs;
-       struct config *conf;
+       struct uevent *merge_uev, *tmp;
 
        vecs = (struct vectors *)trigger_data;
 
-       if (uev_discard(uev->devpath))
-               return 0;
-
        pthread_cleanup_push(config_cleanup, NULL);
        pthread_mutex_lock(&config_lock);
        if (running_state != DAEMON_IDLE &&
@@ -1104,28 +1136,21 @@ uev_trigger (struct uevent * uev, void * trigger_data)
        }
 
        /*
-        * path add/remove event
+        * path add/remove/change event, add/remove maybe merged
         */
-       conf = get_multipath_config();
-       if (filter_devnode(conf->blist_devnode, conf->elist_devnode,
-                          uev->kernel) > 0) {
-               put_multipath_config(conf);
-               goto out;
+       list_for_each_entry_safe(merge_uev, tmp, &uev->merge_node, node) {
+               if (!strncmp(merge_uev->action, "add", 3))
+                       r += uev_add_path(merge_uev, vecs, 0);
+               if (!strncmp(merge_uev->action, "remove", 6))
+                       r += uev_remove_path(merge_uev, vecs, 0);
        }
-       put_multipath_config(conf);
 
-       if (!strncmp(uev->action, "add", 3)) {
-               r = uev_add_path(uev, vecs);
-               goto out;
-       }
-       if (!strncmp(uev->action, "remove", 6)) {
-               r = uev_remove_path(uev, vecs);
-               goto out;
-       }
-       if (!strncmp(uev->action, "change", 6)) {
-               r = uev_update_path(uev, vecs);
-               goto out;
-       }
+       if (!strncmp(uev->action, "add", 3))
+               r += uev_add_path(uev, vecs, 1);
+       if (!strncmp(uev->action, "remove", 6))
+               r += uev_remove_path(uev, vecs, 1);
+       if (!strncmp(uev->action, "change", 6))
+               r += uev_update_path(uev, vecs);
 
 out:
        return r;
@@ -1190,6 +1215,8 @@ uxlsnrloop (void * ap)
        set_handler_callback(LIST+BLACKLIST, cli_list_blacklist);
        set_handler_callback(LIST+DEVICES, cli_list_devices);
        set_handler_callback(LIST+WILDCARDS, cli_list_wildcards);
+       set_handler_callback(RESET+MAPS+STATS, cli_reset_maps_stats);
+       set_handler_callback(RESET+MAP+STATS, cli_reset_map_stats);
        set_handler_callback(ADD+PATH, cli_add_path);
        set_handler_callback(DEL+PATH, cli_del_path);
        set_handler_callback(ADD+MAP, cli_add_map);
@@ -1391,6 +1418,7 @@ retry_count_tick(vector mpvec)
                        mpp->stat_total_queueing_time++;
                        condlog(4, "%s: Retrying.. No active path", mpp->alias);
                        if(--mpp->retry_tick == 0) {
+                               mpp->stat_map_failures++;
                                dm_queue_if_no_path(mpp->alias, 0);
                                condlog(2, "%s: Disable queueing", mpp->alias);
                        }
@@ -1421,7 +1449,8 @@ int update_prio(struct path *pp, int refresh_all)
        }
        oldpriority = pp->priority;
        conf = get_multipath_config();
-       pathinfo(pp, conf, DI_PRIO);
+       if (pp->state != PATH_DOWN)
+               pathinfo(pp, conf, DI_PRIO);
        put_multipath_config(conf);
 
        if (pp->priority == oldpriority)
@@ -1442,6 +1471,92 @@ int update_path_groups(struct multipath *mpp, struct vectors *vecs, int refresh)
        return 0;
 }
 
+void repair_path(struct path * pp)
+{
+       if (pp->state != PATH_DOWN)
+               return;
+
+       checker_repair(&pp->checker);
+       LOG_MSG(1, checker_message(&pp->checker));
+}
+
+static int check_path_reinstate_state(struct path * pp) {
+       struct timespec curr_time;
+       if (!((pp->mpp->san_path_err_threshold > 0) &&
+                               (pp->mpp->san_path_err_forget_rate > 0) &&
+                               (pp->mpp->san_path_err_recovery_time >0))) {
+               return 0;
+       }
+
+       if (pp->disable_reinstate) {
+               /* If we don't know how much time has passed, automatically
+                * reinstate the path, just to be safe. Also, if there are
+                * no other usable paths, reinstate the path
+                */
+               if (clock_gettime(CLOCK_MONOTONIC, &curr_time) != 0 ||
+                               pp->mpp->nr_active == 0) {
+                       condlog(2, "%s : reinstating path early", pp->dev);
+                       goto reinstate_path;
+               }
+               if ((curr_time.tv_sec - pp->dis_reinstate_time ) > pp->mpp->san_path_err_recovery_time) {
+                       condlog(2,"%s : reinstate the path after err recovery time", pp->dev);
+                       goto reinstate_path;
+               }
+               return 1;
+       }
+       /* forget errors on a working path */
+       if ((pp->state == PATH_UP || pp->state == PATH_GHOST) &&
+                       pp->path_failures > 0) {
+               if (pp->san_path_err_forget_rate > 0){
+                       pp->san_path_err_forget_rate--;
+               } else {
+                       /* for every san_path_err_forget_rate number of
+                        * successful path checks decrement path_failures by 1
+                        */
+                       pp->path_failures--;
+                       pp->san_path_err_forget_rate = pp->mpp->san_path_err_forget_rate;
+               }
+               return 0;
+       }
+
+       /* If the path isn't recovering from a failed state, do nothing */
+       if (pp->state != PATH_DOWN && pp->state != PATH_SHAKY &&
+                       pp->state != PATH_TIMEOUT)
+               return 0;
+
+       if (pp->path_failures == 0)
+               pp->san_path_err_forget_rate = pp->mpp->san_path_err_forget_rate;
+
+       pp->path_failures++;
+
+       /* if we don't know the currently time, we don't know how long to
+        * delay the path, so there's no point in checking if we should
+        */
+
+       if (clock_gettime(CLOCK_MONOTONIC, &curr_time) != 0)
+               return 0;
+       /* when path failures has exceeded the san_path_err_threshold
+        * place the path in delayed state till san_path_err_recovery_time
+        * so that the cutomer can rectify the issue within this time. After
+        * the completion of san_path_err_recovery_time it should
+        * automatically reinstate the path
+        */
+       if (pp->path_failures > pp->mpp->san_path_err_threshold) {
+               condlog(2, "%s : hit error threshold. Delaying path reinstatement", pp->dev);
+               pp->dis_reinstate_time = curr_time.tv_sec;
+               pp->disable_reinstate = 1;
+               return 1;
+       } else {
+               return 0;
+       }
+
+reinstate_path:
+       pp->path_failures = 0;
+       pp->disable_reinstate = 0;
+       pp->san_path_err_forget_rate = 0;
+       return 0;
+}
+
 /*
  * Returns '1' if the path has been checked, '-1' if it was blacklisted
  * and '0' otherwise
@@ -1505,6 +1620,12 @@ check_path (struct vectors * vecs, struct path * pp, int ticks)
        } else
                checker_clear_message(&pp->checker);
 
+       if (pp->wwid_changed) {
+               condlog(2, "%s: path wwid has changed. Refusing to use",
+                       pp->dev);
+               newstate = PATH_DOWN;
+       }
+
        if (newstate == PATH_WILD || newstate == PATH_UNCHECKED) {
                condlog(2, "%s: unusable path", pp->dev);
                conf = get_multipath_config();
@@ -1519,7 +1640,7 @@ check_path (struct vectors * vecs, struct path * pp, int ticks)
                        conf = get_multipath_config();
                        ret = pathinfo(pp, conf, DI_ALL | DI_BLACKLIST);
                        if (ret == PATHINFO_OK) {
-                               ev_add_path(pp, vecs);
+                               ev_add_path(pp, vecs, 1);
                                pp->tick = 1;
                        } else if (ret == PATHINFO_SKIPPED) {
                                put_multipath_config(conf);
@@ -1549,9 +1670,15 @@ check_path (struct vectors * vecs, struct path * pp, int ticks)
        if (!pp->mpp)
                return 0;
 
+       if ((newstate == PATH_UP || newstate == PATH_GHOST) &&
+                       check_path_reinstate_state(pp)) {
+               pp->state = PATH_DELAYED;
+               return 1;
+       }
+
        if ((newstate == PATH_UP || newstate == PATH_GHOST) &&
             pp->wait_checks > 0) {
-               if (pp->mpp && pp->mpp->nr_active > 0) {
+               if (pp->mpp->nr_active > 0) {
                        pp->state = PATH_DELAYED;
                        pp->wait_checks--;
                        return 1;
@@ -1574,8 +1701,7 @@ check_path (struct vectors * vecs, struct path * pp, int ticks)
                int oldstate = pp->state;
                pp->state = newstate;
 
-               if (strlen(checker_message(&pp->checker)))
-                       LOG_MSG(1, checker_message(&pp->checker));
+               LOG_MSG(1, checker_message(&pp->checker));
 
                /*
                 * upon state change, reset the checkint
@@ -1585,7 +1711,7 @@ check_path (struct vectors * vecs, struct path * pp, int ticks)
                pp->checkint = conf->checkint;
                put_multipath_config(conf);
 
-               if (newstate == PATH_DOWN || newstate == PATH_SHAKY) {
+               if (newstate == PATH_DOWN || newstate == PATH_SHAKY || newstate == PATH_TIMEOUT) {
                        /*
                         * proactively fail path in the DM
                         */
@@ -1606,6 +1732,7 @@ check_path (struct vectors * vecs, struct path * pp, int ticks)
                        pp->mpp->failback_tick = 0;
 
                        pp->mpp->stat_path_failures++;
+                       repair_path(pp);
                        return 1;
                }
 
@@ -1635,7 +1762,7 @@ check_path (struct vectors * vecs, struct path * pp, int ticks)
                }
                if (!disable_reinstate && reinstate_path(pp, add_active)) {
                        condlog(3, "%s: reload map", pp->dev);
-                       ev_add_path(pp, vecs);
+                       ev_add_path(pp, vecs, 1);
                        pp->tick = 1;
                        return 0;
                }
@@ -1658,7 +1785,7 @@ check_path (struct vectors * vecs, struct path * pp, int ticks)
                        /* Clear IO errors */
                        if (reinstate_path(pp, 0)) {
                                condlog(3, "%s: reload map", pp->dev);
-                               ev_add_path(pp, vecs);
+                               ev_add_path(pp, vecs, 1);
                                pp->tick = 1;
                                return 0;
                        }
@@ -1686,8 +1813,7 @@ check_path (struct vectors * vecs, struct path * pp, int ticks)
                        pp->tick = pp->checkint;
                }
        }
-       else if (newstate == PATH_DOWN &&
-                strlen(checker_message(&pp->checker))) {
+       else if (newstate == PATH_DOWN) {
                int log_checker_err;
 
                conf = get_multipath_config();
@@ -1700,7 +1826,7 @@ check_path (struct vectors * vecs, struct path * pp, int ticks)
        }
 
        pp->state = newstate;
-
+       repair_path(pp);
 
        if (pp->mpp->wait_for_udev)
                return 1;
@@ -1725,6 +1851,19 @@ check_path (struct vectors * vecs, struct path * pp, int ticks)
        return 1;
 }
 
+static void init_path_check_interval(struct vectors *vecs)
+{
+       struct config *conf;
+       struct path *pp;
+       unsigned int i;
+
+       vector_foreach_slot (vecs->pathvec, pp, i) {
+               conf = get_multipath_config();
+               pp->checkint = conf->checkint;
+               put_multipath_config(conf);
+       }
+}
+
 static void *
 checkerloop (void *ap)
 {
@@ -1733,7 +1872,7 @@ checkerloop (void *ap)
        int count = 0;
        unsigned int i;
        struct itimerval timer_tick_it;
-       struct timeval last_time;
+       struct timespec last_time;
        struct config *conf;
 
        pthread_cleanup_push(rcu_unregister, NULL);
@@ -1742,34 +1881,24 @@ checkerloop (void *ap)
        vecs = (struct vectors *)ap;
        condlog(2, "path checkers start up");
 
-       /*
-        * init the path check interval
-        */
-       vector_foreach_slot (vecs->pathvec, pp, i) {
-               conf = get_multipath_config();
-               pp->checkint = conf->checkint;
-               put_multipath_config(conf);
-       }
-
        /* Tweak start time for initial path check */
-       if (gettimeofday(&last_time, NULL) != 0)
+       if (clock_gettime(CLOCK_MONOTONIC, &last_time) != 0)
                last_time.tv_sec = 0;
        else
                last_time.tv_sec -= 1;
 
        while (1) {
-               struct timeval diff_time, start_time, end_time;
+               struct timespec diff_time, start_time, end_time;
                int num_paths = 0, ticks = 0, signo, strict_timing, rc = 0;
                sigset_t mask;
 
-               if (gettimeofday(&start_time, NULL) != 0)
+               if (clock_gettime(CLOCK_MONOTONIC, &start_time) != 0)
                        start_time.tv_sec = 0;
                if (start_time.tv_sec && last_time.tv_sec) {
-                       timersub(&start_time, &last_time, &diff_time);
+                       timespecsub(&start_time, &last_time, &diff_time);
                        condlog(4, "tick (%lu.%06lu secs)",
-                               diff_time.tv_sec, diff_time.tv_usec);
-                       last_time.tv_sec = start_time.tv_sec;
-                       last_time.tv_usec = start_time.tv_usec;
+                               diff_time.tv_sec, diff_time.tv_nsec / 1000);
+                       last_time = start_time;
                        ticks = diff_time.tv_sec;
                } else {
                        ticks = 1;
@@ -1784,35 +1913,34 @@ checkerloop (void *ap)
                        condlog(4, "timeout waiting for DAEMON_IDLE");
                        continue;
                }
-               if (vecs->pathvec) {
-                       pthread_cleanup_push(cleanup_lock, &vecs->lock);
-                       lock(vecs->lock);
-                       pthread_testcancel();
-                       vector_foreach_slot (vecs->pathvec, pp, i) {
-                               rc = check_path(vecs, pp, ticks);
-                               if (rc < 0) {
-                                       vector_del_slot(vecs->pathvec, i);
-                                       free_path(pp);
-                                       i--;
-                               } else
-                                       num_paths += rc;
-                       }
-                       lock_cleanup_pop(vecs->lock);
-               }
-               if (vecs->mpvec) {
-                       pthread_cleanup_push(cleanup_lock, &vecs->lock);
-                       lock(vecs->lock);
-                       pthread_testcancel();
-                       defered_failback_tick(vecs->mpvec);
-                       retry_count_tick(vecs->mpvec);
-                       missing_uev_wait_tick(vecs);
-                       lock_cleanup_pop(vecs->lock);
+
+               pthread_cleanup_push(cleanup_lock, &vecs->lock);
+               lock(&vecs->lock);
+               pthread_testcancel();
+               vector_foreach_slot (vecs->pathvec, pp, i) {
+                       rc = check_path(vecs, pp, ticks);
+                       if (rc < 0) {
+                               vector_del_slot(vecs->pathvec, i);
+                               free_path(pp);
+                               i--;
+                       } else
+                               num_paths += rc;
                }
+               lock_cleanup_pop(vecs->lock);
+
+               pthread_cleanup_push(cleanup_lock, &vecs->lock);
+               lock(&vecs->lock);
+               pthread_testcancel();
+               defered_failback_tick(vecs->mpvec);
+               retry_count_tick(vecs->mpvec);
+               missing_uev_wait_tick(vecs);
+               lock_cleanup_pop(vecs->lock);
+
                if (count)
                        count--;
                else {
                        pthread_cleanup_push(cleanup_lock, &vecs->lock);
-                       lock(vecs->lock);
+                       lock(&vecs->lock);
                        pthread_testcancel();
                        condlog(4, "map garbage collection");
                        mpvec_garbage_collector(vecs);
@@ -1820,16 +1948,17 @@ checkerloop (void *ap)
                        lock_cleanup_pop(vecs->lock);
                }
 
-               diff_time.tv_usec = 0;
+               diff_time.tv_nsec = 0;
                if (start_time.tv_sec &&
-                   gettimeofday(&end_time, NULL) == 0) {
-                       timersub(&end_time, &start_time, &diff_time);
+                   clock_gettime(CLOCK_MONOTONIC, &end_time) == 0) {
+                       timespecsub(&end_time, &start_time, &diff_time);
                        if (num_paths) {
                                unsigned int max_checkint;
 
                                condlog(3, "checked %d path%s in %lu.%06lu secs",
                                        num_paths, num_paths > 1 ? "s" : "",
-                                       diff_time.tv_sec, diff_time.tv_usec);
+                                       diff_time.tv_sec,
+                                       diff_time.tv_nsec / 1000);
                                conf = get_multipath_config();
                                max_checkint = conf->max_checkint;
                                put_multipath_config(conf);
@@ -1850,10 +1979,10 @@ checkerloop (void *ap)
                else {
                        timer_tick_it.it_interval.tv_sec = 0;
                        timer_tick_it.it_interval.tv_usec = 0;
-                       if (diff_time.tv_usec) {
+                       if (diff_time.tv_nsec) {
                                timer_tick_it.it_value.tv_sec = 0;
                                timer_tick_it.it_value.tv_usec =
-                                       (unsigned long)1000000 - diff_time.tv_usec;
+                                    1000UL * 1000 * 1000 - diff_time.tv_nsec;
                        } else {
                                timer_tick_it.it_value.tv_sec = 1;
                                timer_tick_it.it_value.tv_usec = 0;
@@ -1888,21 +2017,29 @@ configure (struct vectors * vecs, int start_waiters)
        int i, ret;
        struct config *conf;
 
-       if (!vecs->pathvec && !(vecs->pathvec = vector_alloc()))
+       if (!vecs->pathvec && !(vecs->pathvec = vector_alloc())) {
+               condlog(0, "couldn't allocate path vec in configure");
                return 1;
+       }
 
-       if (!vecs->mpvec && !(vecs->mpvec = vector_alloc()))
+       if (!vecs->mpvec && !(vecs->mpvec = vector_alloc())) {
+               condlog(0, "couldn't allocate multipath vec in configure");
                return 1;
+       }
 
-       if (!(mpvec = vector_alloc()))
+       if (!(mpvec = vector_alloc())) {
+               condlog(0, "couldn't allocate new maps vec in configure");
                return 1;
+       }
 
        /*
         * probe for current path (from sysfs) and map (from dm) sets
         */
        ret = path_discovery(vecs->pathvec, DI_ALL);
-       if (ret < 0)
+       if (ret < 0) {
+               condlog(0, "configure failed at path discovery");
                return 1;
+       }
 
        vector_foreach_slot (vecs->pathvec, pp, i){
                conf = get_multipath_config();
@@ -1915,21 +2052,27 @@ configure (struct vectors * vecs, int start_waiters)
                        pp->checkint = conf->checkint;
                put_multipath_config(conf);
        }
-       if (map_discovery(vecs))
+       if (map_discovery(vecs)) {
+               condlog(0, "configure failed at map discovery");
                return 1;
+       }
 
        /*
         * create new set of maps & push changed ones into dm
         */
-       if (coalesce_paths(vecs, mpvec, NULL, 1, CMD_NONE))
+       if (coalesce_paths(vecs, mpvec, NULL, 1, CMD_NONE)) {
+               condlog(0, "configure failed while coalescing paths");
                return 1;
+       }
 
        /*
         * may need to remove some maps which are no longer relevant
         * e.g., due to blacklist changes in conf file
         */
-       if (coalesce_maps(vecs, mpvec))
+       if (coalesce_maps(vecs, mpvec)) {
+               condlog(0, "configure failed while coalescing maps");
                return 1;
+       }
 
        dm_lib_release();
 
@@ -1954,11 +2097,16 @@ configure (struct vectors * vecs, int start_waiters)
         * start dm event waiter threads for these new maps
         */
        vector_foreach_slot(vecs->mpvec, mpp, i) {
-               if (setup_multipath(vecs, mpp))
-                       return 1;
-               if (start_waiters)
-                       if (start_waiter_thread(mpp, vecs))
-                               return 1;
+               if (setup_multipath(vecs, mpp)) {
+                       i--;
+                       continue;
+               }
+               if (start_waiters) {
+                       if (start_waiter_thread(mpp, vecs)) {
+                               remove_map(mpp, vecs, 1);
+                               i--;
+                       }
+               }
        }
        return 0;
 }
@@ -2036,21 +2184,9 @@ init_vecs (void)
        if (!vecs)
                return NULL;
 
-       vecs->lock.mutex =
-               (pthread_mutex_t *)MALLOC(sizeof(pthread_mutex_t));
-
-       if (!vecs->lock.mutex)
-               goto out;
-
-       pthread_mutex_init(vecs->lock.mutex, NULL);
-       vecs->lock.depth = 0;
+       pthread_mutex_init(&vecs->lock.mutex, NULL);
 
        return vecs;
-
-out:
-       FREE(vecs);
-       condlog(0, "failed to init paths");
-       return NULL;
 }
 
 static void *
@@ -2124,18 +2260,15 @@ signal_init(void)
        sigset_t set;
 
        sigemptyset(&set);
-       sigaddset(&set, SIGHUP);
-       sigaddset(&set, SIGUSR1);
        sigaddset(&set, SIGUSR2);
-       sigaddset(&set, SIGALRM);
-       pthread_sigmask(SIG_BLOCK, &set, NULL);
+       pthread_sigmask(SIG_SETMASK, &set, NULL);
 
        signal_set(SIGHUP, sighup);
        signal_set(SIGUSR1, sigusr1);
        signal_set(SIGUSR2, sigusr2);
        signal_set(SIGINT, sigend);
        signal_set(SIGTERM, sigend);
-       signal(SIGPIPE, SIG_IGN);
+       signal_set(SIGPIPE, sigend);
 }
 
 static void
@@ -2211,6 +2344,7 @@ child (void * param)
        int i;
 #ifdef USE_SYSTEMD
        unsigned long checkint;
+       int startup_done = 0;
 #endif
        int rc;
        int pid_fd = -1;
@@ -2221,8 +2355,8 @@ child (void * param)
        signal_init();
        rcu_init();
 
-       setup_thread_attr(&misc_attr, 64 * 1024, 1);
-       setup_thread_attr(&uevent_attr, DEFAULT_UEVENT_STACKSIZE * 1024, 1);
+       setup_thread_attr(&misc_attr, 64 * 1024, 0);
+       setup_thread_attr(&uevent_attr, DEFAULT_UEVENT_STACKSIZE * 1024, 0);
        setup_thread_attr(&waiter_attr, 32 * 1024, 1);
 
        if (logsink == 1) {
@@ -2331,6 +2465,8 @@ child (void * param)
         */
        post_config_state(DAEMON_CONFIGURE);
 
+       init_path_check_interval(vecs);
+
        /*
         * Start uevent listener early to catch events
         */
@@ -2357,10 +2493,6 @@ child (void * param)
        }
        pthread_attr_destroy(&misc_attr);
 
-#ifdef USE_SYSTEMD
-       sd_notify(0, "READY=1");
-#endif
-
        while (running_state != DAEMON_SHUTDOWN) {
                pthread_cleanup_push(config_cleanup, NULL);
                pthread_mutex_lock(&config_lock);
@@ -2371,7 +2503,7 @@ child (void * param)
                pthread_cleanup_pop(1);
                if (running_state == DAEMON_CONFIGURE) {
                        pthread_cleanup_push(cleanup_lock, &vecs->lock);
-                       lock(vecs->lock);
+                       lock(&vecs->lock);
                        pthread_testcancel();
                        if (!need_to_delay_reconfig(vecs)) {
                                reconfigure(vecs);
@@ -2382,37 +2514,40 @@ child (void * param)
                        }
                        lock_cleanup_pop(vecs->lock);
                        post_config_state(DAEMON_IDLE);
+#ifdef USE_SYSTEMD
+                       if (!startup_done) {
+                               sd_notify(0, "READY=1");
+                               startup_done = 1;
+                       }
+#endif
                }
        }
 
-       lock(vecs->lock);
+       lock(&vecs->lock);
        conf = get_multipath_config();
        if (conf->queue_without_daemon == QUE_NO_DAEMON_OFF)
                vector_foreach_slot(vecs->mpvec, mpp, i)
                        dm_queue_if_no_path(mpp->alias, 0);
        put_multipath_config(conf);
        remove_maps_and_stop_waiters(vecs);
-       unlock(vecs->lock);
+       unlock(&vecs->lock);
 
        pthread_cancel(check_thr);
        pthread_cancel(uevent_thr);
        pthread_cancel(uxlsnr_thr);
        pthread_cancel(uevq_thr);
 
-       lock(vecs->lock);
+       pthread_join(check_thr, NULL);
+       pthread_join(uevent_thr, NULL);
+       pthread_join(uxlsnr_thr, NULL);
+       pthread_join(uevq_thr, NULL);
+
+       lock(&vecs->lock);
        free_pathvec(vecs->pathvec, FREE_PATHS);
        vecs->pathvec = NULL;
-       unlock(vecs->lock);
-       /* Now all the waitevent threads will start rushing in. */
-       while (vecs->lock.depth > 0) {
-               sleep (1); /* This is weak. */
-               condlog(3, "Have %d wait event checkers threads to de-alloc,"
-                       " waiting...", vecs->lock.depth);
-       }
-       pthread_mutex_destroy(vecs->lock.mutex);
-       FREE(vecs->lock.mutex);
-       vecs->lock.depth = 0;
-       vecs->lock.mutex = NULL;
+       unlock(&vecs->lock);
+
+       pthread_mutex_destroy(&vecs->lock.mutex);
        FREE(vecs);
        vecs = NULL;
 
@@ -2523,6 +2658,13 @@ main (int argc, char *argv[])
        int foreground = 0;
        struct config *conf;
 
+       ANNOTATE_BENIGN_RACE_SIZED(&multipath_conf, sizeof(multipath_conf),
+                                  "Manipulated through RCU");
+       ANNOTATE_BENIGN_RACE_SIZED(&running_state, sizeof(running_state),
+               "Suppress complaints about unprotected running_state reads");
+       ANNOTATE_BENIGN_RACE_SIZED(&uxsock_timeout, sizeof(uxsock_timeout),
+               "Suppress complaints about this scalar variable");
+
        logsink = 1;
 
        if (getuid() != 0) {
@@ -2536,10 +2678,12 @@ main (int argc, char *argv[])
                        strerror(errno));
        umask(umask(077) | 022);
 
+       pthread_cond_init_mono(&config_cond);
+
        udev = udev_new();
 
        while ((arg = getopt(argc, argv, ":dsv:k::Bn")) != EOF ) {
-       switch(arg) {
+               switch(arg) {
                case 'd':
                        foreground = 1;
                        if (logsink > 0)
@@ -2562,7 +2706,9 @@ main (int argc, char *argv[])
                                exit(1);
                        if (verbosity)
                                conf->verbosity = verbosity;
+                       uxsock_timeout = conf->uxsock_timeout;
                        uxclnt(optarg, uxsock_timeout + 100);
+                       free_config(conf);
                        exit(0);
                case 'B':
                        bindings_read_only = 1;
@@ -2586,6 +2732,7 @@ main (int argc, char *argv[])
                        exit(1);
                if (verbosity)
                        conf->verbosity = verbosity;
+               uxsock_timeout = conf->uxsock_timeout;
                memset(cmd, 0x0, CMDSIZE);
                while (optind < argc) {
                        if (strchr(argv[optind], ' '))
@@ -2596,6 +2743,7 @@ main (int argc, char *argv[])
                }
                c += snprintf(c, s + CMDSIZE - c, "\n");
                uxclnt(s, uxsock_timeout + 100);
+               free_config(conf);
                exit(0);
        }