multipathd: marginal path code fixes
[multipath-tools/.git] / multipathd / main.c
1 /*
2  * Copyright (c) 2004, 2005 Christophe Varoqui
3  * Copyright (c) 2005 Kiyoshi Ueda, NEC
4  * Copyright (c) 2005 Benjamin Marzinski, Redhat
5  * Copyright (c) 2005 Edward Goggin, EMC
6  */
7 #include <unistd.h>
8 #include <sys/stat.h>
9 #include <libdevmapper.h>
10 #include <sys/wait.h>
11 #include <sys/mman.h>
12 #include <sys/types.h>
13 #include <fcntl.h>
14 #include <errno.h>
15 #include <sys/time.h>
16 #include <sys/resource.h>
17 #include <limits.h>
18 #include <linux/oom.h>
19 #include <libudev.h>
20 #include <urcu.h>
21 #ifdef USE_SYSTEMD
22 #include <systemd/sd-daemon.h>
23 #endif
24 #include <semaphore.h>
25 #include <time.h>
26 #include <stdbool.h>
27
28 /*
29  * libmultipath
30  */
31 #include "time-util.h"
32
33 /*
34  * libcheckers
35  */
36 #include "checkers.h"
37
38 #ifdef USE_SYSTEMD
39 static int use_watchdog;
40 #endif
41
42 /*
43  * libmultipath
44  */
45 #include "parser.h"
46 #include "vector.h"
47 #include "memory.h"
48 #include "config.h"
49 #include "util.h"
50 #include "hwtable.h"
51 #include "defaults.h"
52 #include "structs.h"
53 #include "blacklist.h"
54 #include "structs_vec.h"
55 #include "dmparser.h"
56 #include "devmapper.h"
57 #include "sysfs.h"
58 #include "dict.h"
59 #include "discovery.h"
60 #include "debug.h"
61 #include "propsel.h"
62 #include "uevent.h"
63 #include "switchgroup.h"
64 #include "print.h"
65 #include "configure.h"
66 #include "prio.h"
67 #include "wwids.h"
68 #include "pgpolicies.h"
69 #include "uevent.h"
70 #include "log.h"
71
72 #include "mpath_cmd.h"
73 #include "mpath_persist.h"
74
75 #include "prioritizers/alua_rtpg.h"
76
77 #include "main.h"
78 #include "pidfile.h"
79 #include "uxlsnr.h"
80 #include "uxclnt.h"
81 #include "cli.h"
82 #include "cli_handlers.h"
83 #include "lock.h"
84 #include "waiter.h"
85 #include "io_err_stat.h"
86 #include "wwids.h"
87 #include "../third-party/valgrind/drd.h"
88
89 #define FILE_NAME_SIZE 256
90 #define CMDSIZE 160
91
92 #define LOG_MSG(a, b) \
93 do { \
94         if (pp->offline) \
95                 condlog(a, "%s: %s - path offline", pp->mpp->alias, pp->dev); \
96         else if (strlen(b)) \
97                 condlog(a, "%s: %s - %s", pp->mpp->alias, pp->dev, b); \
98 } while(0)
99
100 struct mpath_event_param
101 {
102         char * devname;
103         struct multipath *mpp;
104 };
105
106 int logsink;
107 int uxsock_timeout;
108 int verbosity;
109 int bindings_read_only;
110 int ignore_new_devs;
111 enum daemon_status running_state = DAEMON_INIT;
112 pid_t daemon_pid;
113 pthread_mutex_t config_lock = PTHREAD_MUTEX_INITIALIZER;
114 pthread_cond_t config_cond;
115
116 /*
117  * global copy of vecs for use in sig handlers
118  */
119 struct vectors * gvecs;
120
121 struct udev * udev;
122
123 struct config *multipath_conf;
124
125 /* Local variables */
126 static volatile sig_atomic_t exit_sig;
127 static volatile sig_atomic_t reconfig_sig;
128 static volatile sig_atomic_t log_reset_sig;
129
130 const char *
131 daemon_status(void)
132 {
133         switch (running_state) {
134         case DAEMON_INIT:
135                 return "init";
136         case DAEMON_START:
137                 return "startup";
138         case DAEMON_CONFIGURE:
139                 return "configure";
140         case DAEMON_IDLE:
141                 return "idle";
142         case DAEMON_RUNNING:
143                 return "running";
144         case DAEMON_SHUTDOWN:
145                 return "shutdown";
146         }
147         return NULL;
148 }
149
150 /*
151  * I love you too, systemd ...
152  */
153 const char *
154 sd_notify_status(void)
155 {
156         switch (running_state) {
157         case DAEMON_INIT:
158                 return "STATUS=init";
159         case DAEMON_START:
160                 return "STATUS=startup";
161         case DAEMON_CONFIGURE:
162                 return "STATUS=configure";
163         case DAEMON_IDLE:
164         case DAEMON_RUNNING:
165                 return "STATUS=up";
166         case DAEMON_SHUTDOWN:
167                 return "STATUS=shutdown";
168         }
169         return NULL;
170 }
171
172 #ifdef USE_SYSTEMD
173 static void do_sd_notify(enum daemon_status old_state)
174 {
175         /*
176          * Checkerloop switches back and forth between idle and running state.
177          * No need to tell systemd each time.
178          * These notifications cause a lot of overhead on dbus.
179          */
180         if ((running_state == DAEMON_IDLE || running_state == DAEMON_RUNNING) &&
181             (old_state == DAEMON_IDLE || old_state == DAEMON_RUNNING))
182                 return;
183         sd_notify(0, sd_notify_status());
184 }
185 #endif
186
187 static void config_cleanup(void *arg)
188 {
189         pthread_mutex_unlock(&config_lock);
190 }
191
192 void post_config_state(enum daemon_status state)
193 {
194         pthread_mutex_lock(&config_lock);
195         if (state != running_state) {
196                 enum daemon_status old_state = running_state;
197
198                 running_state = state;
199                 pthread_cond_broadcast(&config_cond);
200 #ifdef USE_SYSTEMD
201                 do_sd_notify(old_state);
202 #endif
203         }
204         pthread_mutex_unlock(&config_lock);
205 }
206
207 int set_config_state(enum daemon_status state)
208 {
209         int rc = 0;
210
211         pthread_cleanup_push(config_cleanup, NULL);
212         pthread_mutex_lock(&config_lock);
213         if (running_state != state) {
214                 enum daemon_status old_state = running_state;
215
216                 if (running_state != DAEMON_IDLE) {
217                         struct timespec ts;
218
219                         clock_gettime(CLOCK_MONOTONIC, &ts);
220                         ts.tv_sec += 1;
221                         rc = pthread_cond_timedwait(&config_cond,
222                                                     &config_lock, &ts);
223                 }
224                 if (!rc) {
225                         running_state = state;
226                         pthread_cond_broadcast(&config_cond);
227 #ifdef USE_SYSTEMD
228                         do_sd_notify(old_state);
229 #endif
230                 }
231         }
232         pthread_cleanup_pop(1);
233         return rc;
234 }
235
236 struct config *get_multipath_config(void)
237 {
238         rcu_read_lock();
239         return rcu_dereference(multipath_conf);
240 }
241
242 void put_multipath_config(struct config *conf)
243 {
244         rcu_read_unlock();
245 }
246
247 static int
248 need_switch_pathgroup (struct multipath * mpp, int refresh)
249 {
250         struct pathgroup * pgp;
251         struct path * pp;
252         unsigned int i, j;
253         struct config *conf;
254
255         if (!mpp || mpp->pgfailback == -FAILBACK_MANUAL)
256                 return 0;
257
258         /*
259          * Refresh path priority values
260          */
261         if (refresh) {
262                 vector_foreach_slot (mpp->pg, pgp, i) {
263                         vector_foreach_slot (pgp->paths, pp, j) {
264                                 conf = get_multipath_config();
265                                 pathinfo(pp, conf, DI_PRIO);
266                                 put_multipath_config(conf);
267                         }
268                 }
269         }
270
271         if (!mpp->pg || VECTOR_SIZE(mpp->paths) == 0)
272                 return 0;
273
274         mpp->bestpg = select_path_group(mpp);
275
276         if (mpp->bestpg != mpp->nextpg)
277                 return 1;
278
279         return 0;
280 }
281
282 static void
283 switch_pathgroup (struct multipath * mpp)
284 {
285         mpp->stat_switchgroup++;
286         dm_switchgroup(mpp->alias, mpp->bestpg);
287         condlog(2, "%s: switch to path group #%i",
288                  mpp->alias, mpp->bestpg);
289 }
290
291 static int
292 coalesce_maps(struct vectors *vecs, vector nmpv)
293 {
294         struct multipath * ompp;
295         vector ompv = vecs->mpvec;
296         unsigned int i, reassign_maps;
297         struct config *conf;
298
299         conf = get_multipath_config();
300         reassign_maps = conf->reassign_maps;
301         put_multipath_config(conf);
302         vector_foreach_slot (ompv, ompp, i) {
303                 condlog(3, "%s: coalesce map", ompp->alias);
304                 if (!find_mp_by_wwid(nmpv, ompp->wwid)) {
305                         /*
306                          * remove all current maps not allowed by the
307                          * current configuration
308                          */
309                         if (dm_flush_map(ompp->alias)) {
310                                 condlog(0, "%s: unable to flush devmap",
311                                         ompp->alias);
312                                 /*
313                                  * may be just because the device is open
314                                  */
315                                 if (setup_multipath(vecs, ompp) != 0) {
316                                         i--;
317                                         continue;
318                                 }
319                                 if (!vector_alloc_slot(nmpv))
320                                         return 1;
321
322                                 vector_set_slot(nmpv, ompp);
323
324                                 vector_del_slot(ompv, i);
325                                 i--;
326                         }
327                         else {
328                                 dm_lib_release();
329                                 condlog(2, "%s devmap removed", ompp->alias);
330                         }
331                 } else if (reassign_maps) {
332                         condlog(3, "%s: Reassign existing device-mapper"
333                                 " devices", ompp->alias);
334                         dm_reassign(ompp->alias);
335                 }
336         }
337         return 0;
338 }
339
340 static void
341 sync_maps_state(vector mpvec)
342 {
343         unsigned int i;
344         struct multipath *mpp;
345
346         vector_foreach_slot (mpvec, mpp, i)
347                 sync_map_state(mpp);
348 }
349
350 static int
351 flush_map(struct multipath * mpp, struct vectors * vecs, int nopaths)
352 {
353         int r;
354
355         if (nopaths)
356                 r = dm_flush_map_nopaths(mpp->alias, mpp->deferred_remove);
357         else
358                 r = dm_flush_map(mpp->alias);
359         /*
360          * clear references to this map before flushing so we can ignore
361          * the spurious uevent we may generate with the dm_flush_map call below
362          */
363         if (r) {
364                 /*
365                  * May not really be an error -- if the map was already flushed
366                  * from the device mapper by dmsetup(8) for instance.
367                  */
368                 if (r == 1)
369                         condlog(0, "%s: can't flush", mpp->alias);
370                 else {
371                         condlog(2, "%s: devmap deferred remove", mpp->alias);
372                         mpp->deferred_remove = DEFERRED_REMOVE_IN_PROGRESS;
373                 }
374                 return r;
375         }
376         else {
377                 dm_lib_release();
378                 condlog(2, "%s: map flushed", mpp->alias);
379         }
380
381         orphan_paths(vecs->pathvec, mpp);
382         remove_map_and_stop_waiter(mpp, vecs, 1);
383
384         return 0;
385 }
386
387 static int
388 uev_add_map (struct uevent * uev, struct vectors * vecs)
389 {
390         char *alias;
391         int major = -1, minor = -1, rc;
392
393         condlog(3, "%s: add map (uevent)", uev->kernel);
394         alias = uevent_get_dm_name(uev);
395         if (!alias) {
396                 condlog(3, "%s: No DM_NAME in uevent", uev->kernel);
397                 major = uevent_get_major(uev);
398                 minor = uevent_get_minor(uev);
399                 alias = dm_mapname(major, minor);
400                 if (!alias) {
401                         condlog(2, "%s: mapname not found for %d:%d",
402                                 uev->kernel, major, minor);
403                         return 1;
404                 }
405         }
406         pthread_cleanup_push(cleanup_lock, &vecs->lock);
407         lock(&vecs->lock);
408         pthread_testcancel();
409         rc = ev_add_map(uev->kernel, alias, vecs);
410         lock_cleanup_pop(vecs->lock);
411         FREE(alias);
412         return rc;
413 }
414
415 int
416 ev_add_map (char * dev, char * alias, struct vectors * vecs)
417 {
418         char * refwwid;
419         struct multipath * mpp;
420         int map_present;
421         int r = 1, delayed_reconfig, reassign_maps;
422         struct config *conf;
423
424         map_present = dm_map_present(alias);
425
426         if (map_present && !dm_is_mpath(alias)) {
427                 condlog(4, "%s: not a multipath map", alias);
428                 return 0;
429         }
430
431         mpp = find_mp_by_alias(vecs->mpvec, alias);
432
433         if (mpp) {
434                 if (mpp->wait_for_udev > 1) {
435                         condlog(2, "%s: performing delayed actions",
436                                 mpp->alias);
437                         if (update_map(mpp, vecs))
438                                 /* setup multipathd removed the map */
439                                 return 1;
440                 }
441                 conf = get_multipath_config();
442                 delayed_reconfig = conf->delayed_reconfig;
443                 reassign_maps = conf->reassign_maps;
444                 put_multipath_config(conf);
445                 if (mpp->wait_for_udev) {
446                         mpp->wait_for_udev = 0;
447                         if (delayed_reconfig &&
448                             !need_to_delay_reconfig(vecs)) {
449                                 condlog(2, "reconfigure (delayed)");
450                                 set_config_state(DAEMON_CONFIGURE);
451                                 return 0;
452                         }
453                 }
454                 /*
455                  * Not really an error -- we generate our own uevent
456                  * if we create a multipath mapped device as a result
457                  * of uev_add_path
458                  */
459                 if (reassign_maps) {
460                         condlog(3, "%s: Reassign existing device-mapper devices",
461                                 alias);
462                         dm_reassign(alias);
463                 }
464                 return 0;
465         }
466         condlog(2, "%s: adding map", alias);
467
468         /*
469          * now we can register the map
470          */
471         if (map_present) {
472                 if ((mpp = add_map_without_path(vecs, alias))) {
473                         sync_map_state(mpp);
474                         condlog(2, "%s: devmap %s registered", alias, dev);
475                         return 0;
476                 } else {
477                         condlog(2, "%s: uev_add_map failed", dev);
478                         return 1;
479                 }
480         }
481         r = get_refwwid(CMD_NONE, dev, DEV_DEVMAP, vecs->pathvec, &refwwid);
482
483         if (refwwid) {
484                 r = coalesce_paths(vecs, NULL, refwwid, FORCE_RELOAD_NONE,
485                                    CMD_NONE);
486                 dm_lib_release();
487         }
488
489         if (!r)
490                 condlog(2, "%s: devmap %s added", alias, dev);
491         else if (r == 2)
492                 condlog(2, "%s: uev_add_map %s blacklisted", alias, dev);
493         else
494                 condlog(0, "%s: uev_add_map %s failed", alias, dev);
495
496         FREE(refwwid);
497         return r;
498 }
499
500 static int
501 uev_remove_map (struct uevent * uev, struct vectors * vecs)
502 {
503         char *alias;
504         int minor;
505         struct multipath *mpp;
506
507         condlog(2, "%s: remove map (uevent)", uev->kernel);
508         alias = uevent_get_dm_name(uev);
509         if (!alias) {
510                 condlog(3, "%s: No DM_NAME in uevent, ignoring", uev->kernel);
511                 return 0;
512         }
513         minor = uevent_get_minor(uev);
514
515         pthread_cleanup_push(cleanup_lock, &vecs->lock);
516         lock(&vecs->lock);
517         pthread_testcancel();
518         mpp = find_mp_by_minor(vecs->mpvec, minor);
519
520         if (!mpp) {
521                 condlog(2, "%s: devmap not registered, can't remove",
522                         uev->kernel);
523                 goto out;
524         }
525         if (strcmp(mpp->alias, alias)) {
526                 condlog(2, "%s: minor number mismatch (map %d, event %d)",
527                         mpp->alias, mpp->dmi->minor, minor);
528                 goto out;
529         }
530
531         orphan_paths(vecs->pathvec, mpp);
532         remove_map_and_stop_waiter(mpp, vecs, 1);
533 out:
534         lock_cleanup_pop(vecs->lock);
535         FREE(alias);
536         return 0;
537 }
538
539 /* Called from CLI handler */
540 int
541 ev_remove_map (char * devname, char * alias, int minor, struct vectors * vecs)
542 {
543         struct multipath * mpp;
544
545         mpp = find_mp_by_minor(vecs->mpvec, minor);
546
547         if (!mpp) {
548                 condlog(2, "%s: devmap not registered, can't remove",
549                         devname);
550                 return 1;
551         }
552         if (strcmp(mpp->alias, alias)) {
553                 condlog(2, "%s: minor number mismatch (map %d, event %d)",
554                         mpp->alias, mpp->dmi->minor, minor);
555                 return 1;
556         }
557         return flush_map(mpp, vecs, 0);
558 }
559
560 static int
561 uev_add_path (struct uevent *uev, struct vectors * vecs, int need_do_map)
562 {
563         struct path *pp;
564         int ret = 0, i;
565         struct config *conf;
566
567         condlog(2, "%s: add path (uevent)", uev->kernel);
568         if (strstr(uev->kernel, "..") != NULL) {
569                 /*
570                  * Don't allow relative device names in the pathvec
571                  */
572                 condlog(0, "%s: path name is invalid", uev->kernel);
573                 return 1;
574         }
575
576         pthread_cleanup_push(cleanup_lock, &vecs->lock);
577         lock(&vecs->lock);
578         pthread_testcancel();
579         pp = find_path_by_dev(vecs->pathvec, uev->kernel);
580         if (pp) {
581                 int r;
582
583                 condlog(0, "%s: spurious uevent, path already in pathvec",
584                         uev->kernel);
585                 if (!pp->mpp && !strlen(pp->wwid)) {
586                         condlog(3, "%s: reinitialize path", uev->kernel);
587                         udev_device_unref(pp->udev);
588                         pp->udev = udev_device_ref(uev->udev);
589                         conf = get_multipath_config();
590                         r = pathinfo(pp, conf,
591                                      DI_ALL | DI_BLACKLIST);
592                         put_multipath_config(conf);
593                         if (r == PATHINFO_OK)
594                                 ret = ev_add_path(pp, vecs, need_do_map);
595                         else if (r == PATHINFO_SKIPPED) {
596                                 condlog(3, "%s: remove blacklisted path",
597                                         uev->kernel);
598                                 i = find_slot(vecs->pathvec, (void *)pp);
599                                 if (i != -1)
600                                         vector_del_slot(vecs->pathvec, i);
601                                 free_path(pp);
602                         } else {
603                                 condlog(0, "%s: failed to reinitialize path",
604                                         uev->kernel);
605                                 ret = 1;
606                         }
607                 }
608         }
609         lock_cleanup_pop(vecs->lock);
610         if (pp)
611                 return ret;
612
613         /*
614          * get path vital state
615          */
616         conf = get_multipath_config();
617         ret = alloc_path_with_pathinfo(conf, uev->udev,
618                                        uev->wwid, DI_ALL, &pp);
619         put_multipath_config(conf);
620         if (!pp) {
621                 if (ret == PATHINFO_SKIPPED)
622                         return 0;
623                 condlog(3, "%s: failed to get path info", uev->kernel);
624                 return 1;
625         }
626         pthread_cleanup_push(cleanup_lock, &vecs->lock);
627         lock(&vecs->lock);
628         pthread_testcancel();
629         ret = store_path(vecs->pathvec, pp);
630         if (!ret) {
631                 conf = get_multipath_config();
632                 pp->checkint = conf->checkint;
633                 put_multipath_config(conf);
634                 ret = ev_add_path(pp, vecs, need_do_map);
635         } else {
636                 condlog(0, "%s: failed to store path info, "
637                         "dropping event",
638                         uev->kernel);
639                 free_path(pp);
640                 ret = 1;
641         }
642         lock_cleanup_pop(vecs->lock);
643         return ret;
644 }
645
646 /*
647  * returns:
648  * 0: added
649  * 1: error
650  */
651 int
652 ev_add_path (struct path * pp, struct vectors * vecs, int need_do_map)
653 {
654         struct multipath * mpp;
655         char params[PARAMS_SIZE] = {0};
656         int retries = 3;
657         int start_waiter = 0;
658         int ret;
659
660         /*
661          * need path UID to go any further
662          */
663         if (strlen(pp->wwid) == 0) {
664                 condlog(0, "%s: failed to get path uid", pp->dev);
665                 goto fail; /* leave path added to pathvec */
666         }
667         mpp = find_mp_by_wwid(vecs->mpvec, pp->wwid);
668         if (mpp && mpp->wait_for_udev &&
669             (pathcount(mpp, PATH_UP) > 0 ||
670              (pathcount(mpp, PATH_GHOST) > 0 && pp->tpgs != TPGS_IMPLICIT &&
671               mpp->ghost_delay_tick <= 0))) {
672                 /* if wait_for_udev is set and valid paths exist */
673                 condlog(2, "%s: delaying path addition until %s is fully initialized", pp->dev, mpp->alias);
674                 mpp->wait_for_udev = 2;
675                 orphan_path(pp, "waiting for create to complete");
676                 return 0;
677         }
678
679         pp->mpp = mpp;
680 rescan:
681         if (mpp) {
682                 if (pp->size && mpp->size != pp->size) {
683                         condlog(0, "%s: failed to add new path %s, "
684                                 "device size mismatch",
685                                 mpp->alias, pp->dev);
686                         int i = find_slot(vecs->pathvec, (void *)pp);
687                         if (i != -1)
688                                 vector_del_slot(vecs->pathvec, i);
689                         free_path(pp);
690                         return 1;
691                 }
692
693                 condlog(4,"%s: adopting all paths for path %s",
694                         mpp->alias, pp->dev);
695                 if (adopt_paths(vecs->pathvec, mpp))
696                         goto fail; /* leave path added to pathvec */
697
698                 verify_paths(mpp, vecs);
699                 mpp->action = ACT_RELOAD;
700                 extract_hwe_from_path(mpp);
701         } else {
702                 if (!should_multipath(pp, vecs->pathvec)) {
703                         orphan_path(pp, "only one path");
704                         return 0;
705                 }
706                 condlog(4,"%s: creating new map", pp->dev);
707                 if ((mpp = add_map_with_path(vecs, pp, 1))) {
708                         mpp->action = ACT_CREATE;
709                         /*
710                          * We don't depend on ACT_CREATE, as domap will
711                          * set it to ACT_NOTHING when complete.
712                          */
713                         start_waiter = 1;
714                 }
715                 if (!start_waiter)
716                         goto fail; /* leave path added to pathvec */
717         }
718
719         /* persistent reservation check*/
720         mpath_pr_event_handle(pp);
721
722         if (!need_do_map)
723                 return 0;
724
725         if (!dm_map_present(mpp->alias)) {
726                 mpp->action = ACT_CREATE;
727                 start_waiter = 1;
728         }
729         /*
730          * push the map to the device-mapper
731          */
732         if (setup_map(mpp, params, PARAMS_SIZE)) {
733                 condlog(0, "%s: failed to setup map for addition of new "
734                         "path %s", mpp->alias, pp->dev);
735                 goto fail_map;
736         }
737         /*
738          * reload the map for the multipath mapped device
739          */
740 retry:
741         ret = domap(mpp, params, 1);
742         if (ret <= 0) {
743                 if (ret < 0 && retries-- > 0) {
744                         condlog(0, "%s: retry domap for addition of new "
745                                 "path %s", mpp->alias, pp->dev);
746                         sleep(1);
747                         goto retry;
748                 }
749                 condlog(0, "%s: failed in domap for addition of new "
750                         "path %s", mpp->alias, pp->dev);
751                 /*
752                  * deal with asynchronous uevents :((
753                  */
754                 if (mpp->action == ACT_RELOAD && retries-- > 0) {
755                         condlog(0, "%s: ev_add_path sleep", mpp->alias);
756                         sleep(1);
757                         update_mpp_paths(mpp, vecs->pathvec);
758                         goto rescan;
759                 }
760                 else if (mpp->action == ACT_RELOAD)
761                         condlog(0, "%s: giving up reload", mpp->alias);
762                 else
763                         goto fail_map;
764         }
765         dm_lib_release();
766
767         /*
768          * update our state from kernel regardless of create or reload
769          */
770         if (setup_multipath(vecs, mpp))
771                 goto fail; /* if setup_multipath fails, it removes the map */
772
773         sync_map_state(mpp);
774
775         if ((mpp->action == ACT_CREATE ||
776              (mpp->action == ACT_NOTHING && start_waiter && !mpp->waiter)) &&
777             start_waiter_thread(mpp, vecs))
778                         goto fail_map;
779
780         if (retries >= 0) {
781                 condlog(2, "%s [%s]: path added to devmap %s",
782                         pp->dev, pp->dev_t, mpp->alias);
783                 return 0;
784         } else
785                 goto fail;
786
787 fail_map:
788         remove_map(mpp, vecs, 1);
789 fail:
790         orphan_path(pp, "failed to add path");
791         return 1;
792 }
793
794 static int
795 uev_remove_path (struct uevent *uev, struct vectors * vecs, int need_do_map)
796 {
797         struct path *pp;
798         int ret;
799
800         condlog(2, "%s: remove path (uevent)", uev->kernel);
801         pthread_cleanup_push(cleanup_lock, &vecs->lock);
802         lock(&vecs->lock);
803         pthread_testcancel();
804         pp = find_path_by_dev(vecs->pathvec, uev->kernel);
805         if (pp)
806                 ret = ev_remove_path(pp, vecs, need_do_map);
807         lock_cleanup_pop(vecs->lock);
808         if (!pp) {
809                 /* Not an error; path might have been purged earlier */
810                 condlog(0, "%s: path already removed", uev->kernel);
811                 return 0;
812         }
813         return ret;
814 }
815
816 int
817 ev_remove_path (struct path *pp, struct vectors * vecs, int need_do_map)
818 {
819         struct multipath * mpp;
820         int i, retval = 0;
821         char params[PARAMS_SIZE] = {0};
822
823         /*
824          * avoid referring to the map of an orphaned path
825          */
826         if ((mpp = pp->mpp)) {
827                 /*
828                  * transform the mp->pg vector of vectors of paths
829                  * into a mp->params string to feed the device-mapper
830                  */
831                 if (update_mpp_paths(mpp, vecs->pathvec)) {
832                         condlog(0, "%s: failed to update paths",
833                                 mpp->alias);
834                         goto fail;
835                 }
836                 if ((i = find_slot(mpp->paths, (void *)pp)) != -1)
837                         vector_del_slot(mpp->paths, i);
838
839                 /*
840                  * remove the map IFF removing the last path
841                  */
842                 if (VECTOR_SIZE(mpp->paths) == 0) {
843                         char alias[WWID_SIZE];
844
845                         /*
846                          * flush_map will fail if the device is open
847                          */
848                         strncpy(alias, mpp->alias, WWID_SIZE);
849                         if (mpp->flush_on_last_del == FLUSH_ENABLED) {
850                                 condlog(2, "%s Last path deleted, disabling queueing", mpp->alias);
851                                 mpp->retry_tick = 0;
852                                 mpp->no_path_retry = NO_PATH_RETRY_FAIL;
853                                 mpp->disable_queueing = 1;
854                                 mpp->stat_map_failures++;
855                                 dm_queue_if_no_path(mpp->alias, 0);
856                         }
857                         if (!flush_map(mpp, vecs, 1)) {
858                                 condlog(2, "%s: removed map after"
859                                         " removing all paths",
860                                         alias);
861                                 retval = 0;
862                                 goto out;
863                         }
864                         /*
865                          * Not an error, continue
866                          */
867                 }
868
869                 if (setup_map(mpp, params, PARAMS_SIZE)) {
870                         condlog(0, "%s: failed to setup map for"
871                                 " removal of path %s", mpp->alias, pp->dev);
872                         goto fail;
873                 }
874
875                 if (mpp->wait_for_udev) {
876                         mpp->wait_for_udev = 2;
877                         goto out;
878                 }
879
880                 if (!need_do_map)
881                         goto out;
882                 /*
883                  * reload the map
884                  */
885                 mpp->action = ACT_RELOAD;
886                 if (domap(mpp, params, 1) <= 0) {
887                         condlog(0, "%s: failed in domap for "
888                                 "removal of path %s",
889                                 mpp->alias, pp->dev);
890                         retval = 1;
891                 } else {
892                         /*
893                          * update our state from kernel
894                          */
895                         if (setup_multipath(vecs, mpp))
896                                 return 1;
897                         sync_map_state(mpp);
898
899                         condlog(2, "%s [%s]: path removed from map %s",
900                                 pp->dev, pp->dev_t, mpp->alias);
901                 }
902         }
903
904 out:
905         if ((i = find_slot(vecs->pathvec, (void *)pp)) != -1)
906                 vector_del_slot(vecs->pathvec, i);
907
908         free_path(pp);
909
910         return retval;
911
912 fail:
913         remove_map_and_stop_waiter(mpp, vecs, 1);
914         return 1;
915 }
916
917 static int
918 uev_update_path (struct uevent *uev, struct vectors * vecs)
919 {
920         int ro, retval = 0;
921         struct path * pp;
922         struct config *conf;
923         int disable_changed_wwids;
924         int needs_reinit = 0;
925
926         conf = get_multipath_config();
927         disable_changed_wwids = conf->disable_changed_wwids;
928         put_multipath_config(conf);
929
930         ro = uevent_get_disk_ro(uev);
931
932         pthread_cleanup_push(cleanup_lock, &vecs->lock);
933         lock(&vecs->lock);
934         pthread_testcancel();
935
936         pp = find_path_by_dev(vecs->pathvec, uev->kernel);
937         if (pp) {
938                 struct multipath *mpp = pp->mpp;
939
940                 if (disable_changed_wwids &&
941                     (strlen(pp->wwid) || pp->wwid_changed)) {
942                         char wwid[WWID_SIZE];
943
944                         strcpy(wwid, pp->wwid);
945                         get_uid(pp, pp->state, uev->udev);
946                         if (strcmp(wwid, pp->wwid) != 0) {
947                                 condlog(0, "%s: path wwid changed from '%s' to '%s'. disallowing", uev->kernel, wwid, pp->wwid);
948                                 strcpy(pp->wwid, wwid);
949                                 if (!pp->wwid_changed) {
950                                         pp->wwid_changed = 1;
951                                         pp->tick = 1;
952                                         if (pp->mpp)
953                                                 dm_fail_path(pp->mpp->alias, pp->dev_t);
954                                 }
955                                 goto out;
956                         } else
957                                 pp->wwid_changed = 0;
958                 }
959
960                 if (pp->initialized == INIT_REQUESTED_UDEV)
961                         needs_reinit = 1;
962                 else if (mpp && ro >= 0) {
963                         condlog(2, "%s: update path write_protect to '%d' (uevent)", uev->kernel, ro);
964
965                         if (mpp->wait_for_udev)
966                                 mpp->wait_for_udev = 2;
967                         else {
968                                 if (ro == 1)
969                                         pp->mpp->force_readonly = 1;
970                                 retval = reload_map(vecs, mpp, 0, 1);
971                                 pp->mpp->force_readonly = 0;
972                                 condlog(2, "%s: map %s reloaded (retval %d)",
973                                         uev->kernel, mpp->alias, retval);
974                         }
975                 }
976         }
977 out:
978         lock_cleanup_pop(vecs->lock);
979         if (!pp) {
980                 /* If the path is blacklisted, print a debug/non-default verbosity message. */
981                 if (uev->udev) {
982                         int flag = DI_SYSFS | DI_WWID;
983
984                         conf = get_multipath_config();
985                         retval = alloc_path_with_pathinfo(conf, uev->udev, uev->wwid, flag, NULL);
986                         put_multipath_config(conf);
987
988                         if (retval == PATHINFO_SKIPPED) {
989                                 condlog(3, "%s: spurious uevent, path is blacklisted", uev->kernel);
990                                 return 0;
991                         }
992                 }
993
994                 condlog(0, "%s: spurious uevent, path not found", uev->kernel);
995         }
996         if (needs_reinit)
997                 retval = uev_add_path(uev, vecs, 1);
998         return retval;
999 }
1000
1001 static int
1002 uev_pathfail_check(struct uevent *uev, struct vectors *vecs)
1003 {
1004         char *action = NULL, *devt = NULL;
1005         struct path *pp;
1006         int r = 1;
1007
1008         action = uevent_get_dm_action(uev);
1009         if (!action)
1010                 return 1;
1011         if (strncmp(action, "PATH_FAILED", 11))
1012                 goto out;
1013         devt = uevent_get_dm_path(uev);
1014         if (!devt) {
1015                 condlog(3, "%s: No DM_PATH in uevent", uev->kernel);
1016                 goto out;
1017         }
1018
1019         pthread_cleanup_push(cleanup_lock, &vecs->lock);
1020         lock(&vecs->lock);
1021         pthread_testcancel();
1022         pp = find_path_by_devt(vecs->pathvec, devt);
1023     if (!pp)
1024         goto out_lock;
1025         r = io_err_stat_handle_pathfail(pp);
1026         if (r)
1027                 condlog(3, "io_err_stat: %s: cannot handle pathfail uevent",
1028                                 pp->dev);
1029 out_lock:
1030         lock_cleanup_pop(vecs->lock);
1031         FREE(devt);
1032         FREE(action);
1033         return r;
1034 out:
1035         FREE(action);
1036         return 1;
1037 }
1038
1039 static int
1040 map_discovery (struct vectors * vecs)
1041 {
1042         struct multipath * mpp;
1043         unsigned int i;
1044
1045         if (dm_get_maps(vecs->mpvec))
1046                 return 1;
1047
1048         vector_foreach_slot (vecs->mpvec, mpp, i)
1049                 if (update_multipath_table(mpp, vecs->pathvec, 1) ||
1050                     update_multipath_status(mpp)) {
1051                         remove_map(mpp, vecs, 1);
1052                         i--;
1053                 }
1054
1055         return 0;
1056 }
1057
1058 int
1059 uxsock_trigger (char * str, char ** reply, int * len, bool is_root,
1060                 void * trigger_data)
1061 {
1062         struct vectors * vecs;
1063         int r;
1064
1065         *reply = NULL;
1066         *len = 0;
1067         vecs = (struct vectors *)trigger_data;
1068
1069         if ((str != NULL) && (is_root == false) &&
1070             (strncmp(str, "list", strlen("list")) != 0) &&
1071             (strncmp(str, "show", strlen("show")) != 0)) {
1072                 *reply = STRDUP("permission deny: need to be root");
1073                 if (*reply)
1074                         *len = strlen(*reply) + 1;
1075                 return 1;
1076         }
1077
1078         r = parse_cmd(str, reply, len, vecs, uxsock_timeout / 1000);
1079
1080         if (r > 0) {
1081                 if (r == ETIMEDOUT)
1082                         *reply = STRDUP("timeout\n");
1083                 else
1084                         *reply = STRDUP("fail\n");
1085                 if (*reply)
1086                         *len = strlen(*reply) + 1;
1087                 r = 1;
1088         }
1089         else if (!r && *len == 0) {
1090                 *reply = STRDUP("ok\n");
1091                 if (*reply)
1092                         *len = strlen(*reply) + 1;
1093                 r = 0;
1094         }
1095         /* else if (r < 0) leave *reply alone */
1096
1097         return r;
1098 }
1099
1100 int
1101 uev_trigger (struct uevent * uev, void * trigger_data)
1102 {
1103         int r = 0;
1104         struct vectors * vecs;
1105         struct uevent *merge_uev, *tmp;
1106
1107         vecs = (struct vectors *)trigger_data;
1108
1109         pthread_cleanup_push(config_cleanup, NULL);
1110         pthread_mutex_lock(&config_lock);
1111         if (running_state != DAEMON_IDLE &&
1112             running_state != DAEMON_RUNNING)
1113                 pthread_cond_wait(&config_cond, &config_lock);
1114         pthread_cleanup_pop(1);
1115
1116         if (running_state == DAEMON_SHUTDOWN)
1117                 return 0;
1118
1119         /*
1120          * device map event
1121          * Add events are ignored here as the tables
1122          * are not fully initialised then.
1123          */
1124         if (!strncmp(uev->kernel, "dm-", 3)) {
1125                 if (!strncmp(uev->action, "change", 6)) {
1126                         r = uev_add_map(uev, vecs);
1127
1128                         /*
1129                          * the kernel-side dm-mpath issues a PATH_FAILED event
1130                          * when it encounters a path IO error. It is reason-
1131                          * able be the entry of path IO error accounting pro-
1132                          * cess.
1133                          */
1134                         uev_pathfail_check(uev, vecs);
1135                         goto out;
1136                 }
1137                 if (!strncmp(uev->action, "remove", 6)) {
1138                         r = uev_remove_map(uev, vecs);
1139                         goto out;
1140                 }
1141                 goto out;
1142         }
1143
1144         /*
1145          * path add/remove/change event, add/remove maybe merged
1146          */
1147         list_for_each_entry_safe(merge_uev, tmp, &uev->merge_node, node) {
1148                 if (!strncmp(merge_uev->action, "add", 3))
1149                         r += uev_add_path(merge_uev, vecs, 0);
1150                 if (!strncmp(merge_uev->action, "remove", 6))
1151                         r += uev_remove_path(merge_uev, vecs, 0);
1152         }
1153
1154         if (!strncmp(uev->action, "add", 3))
1155                 r += uev_add_path(uev, vecs, 1);
1156         if (!strncmp(uev->action, "remove", 6))
1157                 r += uev_remove_path(uev, vecs, 1);
1158         if (!strncmp(uev->action, "change", 6))
1159                 r += uev_update_path(uev, vecs);
1160
1161 out:
1162         return r;
1163 }
1164
1165 static void rcu_unregister(void *param)
1166 {
1167         rcu_unregister_thread();
1168 }
1169
1170 static void *
1171 ueventloop (void * ap)
1172 {
1173         struct udev *udev = ap;
1174
1175         pthread_cleanup_push(rcu_unregister, NULL);
1176         rcu_register_thread();
1177         if (uevent_listen(udev))
1178                 condlog(0, "error starting uevent listener");
1179         pthread_cleanup_pop(1);
1180         return NULL;
1181 }
1182
1183 static void *
1184 uevqloop (void * ap)
1185 {
1186         pthread_cleanup_push(rcu_unregister, NULL);
1187         rcu_register_thread();
1188         if (uevent_dispatch(&uev_trigger, ap))
1189                 condlog(0, "error starting uevent dispatcher");
1190         pthread_cleanup_pop(1);
1191         return NULL;
1192 }
1193 static void *
1194 uxlsnrloop (void * ap)
1195 {
1196         if (cli_init()) {
1197                 condlog(1, "Failed to init uxsock listener");
1198                 return NULL;
1199         }
1200         pthread_cleanup_push(rcu_unregister, NULL);
1201         rcu_register_thread();
1202         set_handler_callback(LIST+PATHS, cli_list_paths);
1203         set_handler_callback(LIST+PATHS+FMT, cli_list_paths_fmt);
1204         set_handler_callback(LIST+PATHS+RAW+FMT, cli_list_paths_raw);
1205         set_handler_callback(LIST+PATH, cli_list_path);
1206         set_handler_callback(LIST+MAPS, cli_list_maps);
1207         set_unlocked_handler_callback(LIST+STATUS, cli_list_status);
1208         set_unlocked_handler_callback(LIST+DAEMON, cli_list_daemon);
1209         set_handler_callback(LIST+MAPS+STATUS, cli_list_maps_status);
1210         set_handler_callback(LIST+MAPS+STATS, cli_list_maps_stats);
1211         set_handler_callback(LIST+MAPS+FMT, cli_list_maps_fmt);
1212         set_handler_callback(LIST+MAPS+RAW+FMT, cli_list_maps_raw);
1213         set_handler_callback(LIST+MAPS+TOPOLOGY, cli_list_maps_topology);
1214         set_handler_callback(LIST+TOPOLOGY, cli_list_maps_topology);
1215         set_handler_callback(LIST+MAPS+JSON, cli_list_maps_json);
1216         set_handler_callback(LIST+MAP+TOPOLOGY, cli_list_map_topology);
1217         set_handler_callback(LIST+MAP+FMT, cli_list_map_fmt);
1218         set_handler_callback(LIST+MAP+RAW+FMT, cli_list_map_fmt);
1219         set_handler_callback(LIST+MAP+JSON, cli_list_map_json);
1220         set_handler_callback(LIST+CONFIG, cli_list_config);
1221         set_handler_callback(LIST+BLACKLIST, cli_list_blacklist);
1222         set_handler_callback(LIST+DEVICES, cli_list_devices);
1223         set_handler_callback(LIST+WILDCARDS, cli_list_wildcards);
1224         set_handler_callback(RESET+MAPS+STATS, cli_reset_maps_stats);
1225         set_handler_callback(RESET+MAP+STATS, cli_reset_map_stats);
1226         set_handler_callback(ADD+PATH, cli_add_path);
1227         set_handler_callback(DEL+PATH, cli_del_path);
1228         set_handler_callback(ADD+MAP, cli_add_map);
1229         set_handler_callback(DEL+MAP, cli_del_map);
1230         set_handler_callback(SWITCH+MAP+GROUP, cli_switch_group);
1231         set_unlocked_handler_callback(RECONFIGURE, cli_reconfigure);
1232         set_handler_callback(SUSPEND+MAP, cli_suspend);
1233         set_handler_callback(RESUME+MAP, cli_resume);
1234         set_handler_callback(RESIZE+MAP, cli_resize);
1235         set_handler_callback(RELOAD+MAP, cli_reload);
1236         set_handler_callback(RESET+MAP, cli_reassign);
1237         set_handler_callback(REINSTATE+PATH, cli_reinstate);
1238         set_handler_callback(FAIL+PATH, cli_fail);
1239         set_handler_callback(DISABLEQ+MAP, cli_disable_queueing);
1240         set_handler_callback(RESTOREQ+MAP, cli_restore_queueing);
1241         set_handler_callback(DISABLEQ+MAPS, cli_disable_all_queueing);
1242         set_handler_callback(RESTOREQ+MAPS, cli_restore_all_queueing);
1243         set_unlocked_handler_callback(QUIT, cli_quit);
1244         set_unlocked_handler_callback(SHUTDOWN, cli_shutdown);
1245         set_handler_callback(GETPRSTATUS+MAP, cli_getprstatus);
1246         set_handler_callback(SETPRSTATUS+MAP, cli_setprstatus);
1247         set_handler_callback(UNSETPRSTATUS+MAP, cli_unsetprstatus);
1248         set_handler_callback(FORCEQ+DAEMON, cli_force_no_daemon_q);
1249         set_handler_callback(RESTOREQ+DAEMON, cli_restore_no_daemon_q);
1250         set_handler_callback(GETPRKEY+MAP, cli_getprkey);
1251         set_handler_callback(SETPRKEY+MAP+KEY, cli_setprkey);
1252         set_handler_callback(UNSETPRKEY+MAP, cli_unsetprkey);
1253
1254         umask(077);
1255         uxsock_listen(&uxsock_trigger, ap);
1256         pthread_cleanup_pop(1);
1257         return NULL;
1258 }
1259
1260 void
1261 exit_daemon (void)
1262 {
1263         post_config_state(DAEMON_SHUTDOWN);
1264 }
1265
1266 static void
1267 fail_path (struct path * pp, int del_active)
1268 {
1269         if (!pp->mpp)
1270                 return;
1271
1272         condlog(2, "checker failed path %s in map %s",
1273                  pp->dev_t, pp->mpp->alias);
1274
1275         dm_fail_path(pp->mpp->alias, pp->dev_t);
1276         if (del_active)
1277                 update_queue_mode_del_path(pp->mpp);
1278 }
1279
1280 /*
1281  * caller must have locked the path list before calling that function
1282  */
1283 static int
1284 reinstate_path (struct path * pp, int add_active)
1285 {
1286         int ret = 0;
1287
1288         if (!pp->mpp)
1289                 return 0;
1290
1291         if (dm_reinstate_path(pp->mpp->alias, pp->dev_t)) {
1292                 condlog(0, "%s: reinstate failed", pp->dev_t);
1293                 ret = 1;
1294         } else {
1295                 condlog(2, "%s: reinstated", pp->dev_t);
1296                 if (add_active)
1297                         update_queue_mode_add_path(pp->mpp);
1298         }
1299         return ret;
1300 }
1301
1302 static void
1303 enable_group(struct path * pp)
1304 {
1305         struct pathgroup * pgp;
1306
1307         /*
1308          * if path is added through uev_add_path, pgindex can be unset.
1309          * next update_strings() will set it, upon map reload event.
1310          *
1311          * we can safely return here, because upon map reload, all
1312          * PG will be enabled.
1313          */
1314         if (!pp->mpp->pg || !pp->pgindex)
1315                 return;
1316
1317         pgp = VECTOR_SLOT(pp->mpp->pg, pp->pgindex - 1);
1318
1319         if (pgp->status == PGSTATE_DISABLED) {
1320                 condlog(2, "%s: enable group #%i", pp->mpp->alias, pp->pgindex);
1321                 dm_enablegroup(pp->mpp->alias, pp->pgindex);
1322         }
1323 }
1324
1325 static void
1326 mpvec_garbage_collector (struct vectors * vecs)
1327 {
1328         struct multipath * mpp;
1329         unsigned int i;
1330
1331         if (!vecs->mpvec)
1332                 return;
1333
1334         vector_foreach_slot (vecs->mpvec, mpp, i) {
1335                 if (mpp && mpp->alias && !dm_map_present(mpp->alias)) {
1336                         condlog(2, "%s: remove dead map", mpp->alias);
1337                         remove_map_and_stop_waiter(mpp, vecs, 1);
1338                         i--;
1339                 }
1340         }
1341 }
1342
1343 /* This is called after a path has started working again. It the multipath
1344  * device for this path uses the followover failback type, and this is the
1345  * best pathgroup, and this is the first path in the pathgroup to come back
1346  * up, then switch to this pathgroup */
1347 static int
1348 followover_should_failback(struct path * pp)
1349 {
1350         struct pathgroup * pgp;
1351         struct path *pp1;
1352         int i;
1353
1354         if (pp->mpp->pgfailback != -FAILBACK_FOLLOWOVER ||
1355             !pp->mpp->pg || !pp->pgindex ||
1356             pp->pgindex != pp->mpp->bestpg)
1357                 return 0;
1358
1359         pgp = VECTOR_SLOT(pp->mpp->pg, pp->pgindex - 1);
1360         vector_foreach_slot(pgp->paths, pp1, i) {
1361                 if (pp1 == pp)
1362                         continue;
1363                 if (pp1->chkrstate != PATH_DOWN && pp1->chkrstate != PATH_SHAKY)
1364                         return 0;
1365         }
1366         return 1;
1367 }
1368
1369 static void
1370 missing_uev_wait_tick(struct vectors *vecs)
1371 {
1372         struct multipath * mpp;
1373         unsigned int i;
1374         int timed_out = 0, delayed_reconfig;
1375         struct config *conf;
1376
1377         vector_foreach_slot (vecs->mpvec, mpp, i) {
1378                 if (mpp->wait_for_udev && --mpp->uev_wait_tick <= 0) {
1379                         timed_out = 1;
1380                         condlog(0, "%s: timeout waiting on creation uevent. enabling reloads", mpp->alias);
1381                         if (mpp->wait_for_udev > 1 && update_map(mpp, vecs)) {
1382                                 /* update_map removed map */
1383                                 i--;
1384                                 continue;
1385                         }
1386                         mpp->wait_for_udev = 0;
1387                 }
1388         }
1389
1390         conf = get_multipath_config();
1391         delayed_reconfig = conf->delayed_reconfig;
1392         put_multipath_config(conf);
1393         if (timed_out && delayed_reconfig &&
1394             !need_to_delay_reconfig(vecs)) {
1395                 condlog(2, "reconfigure (delayed)");
1396                 set_config_state(DAEMON_CONFIGURE);
1397         }
1398 }
1399
1400 static void
1401 ghost_delay_tick(struct vectors *vecs)
1402 {
1403         struct multipath * mpp;
1404         unsigned int i;
1405
1406         vector_foreach_slot (vecs->mpvec, mpp, i) {
1407                 if (mpp->ghost_delay_tick <= 0)
1408                         continue;
1409                 if (--mpp->ghost_delay_tick <= 0) {
1410                         condlog(0, "%s: timed out waiting for active path",
1411                                 mpp->alias);
1412                         mpp->force_udev_reload = 1;
1413                         if (update_map(mpp, vecs) != 0) {
1414                                 /* update_map removed map */
1415                                 i--;
1416                                 continue;
1417                         }
1418                 }
1419         }
1420 }
1421
1422 static void
1423 defered_failback_tick (vector mpvec)
1424 {
1425         struct multipath * mpp;
1426         unsigned int i;
1427
1428         vector_foreach_slot (mpvec, mpp, i) {
1429                 /*
1430                  * defered failback getting sooner
1431                  */
1432                 if (mpp->pgfailback > 0 && mpp->failback_tick > 0) {
1433                         mpp->failback_tick--;
1434
1435                         if (!mpp->failback_tick && need_switch_pathgroup(mpp, 1))
1436                                 switch_pathgroup(mpp);
1437                 }
1438         }
1439 }
1440
1441 static void
1442 retry_count_tick(vector mpvec)
1443 {
1444         struct multipath *mpp;
1445         unsigned int i;
1446
1447         vector_foreach_slot (mpvec, mpp, i) {
1448                 if (mpp->retry_tick > 0) {
1449                         mpp->stat_total_queueing_time++;
1450                         condlog(4, "%s: Retrying.. No active path", mpp->alias);
1451                         if(--mpp->retry_tick == 0) {
1452                                 mpp->stat_map_failures++;
1453                                 dm_queue_if_no_path(mpp->alias, 0);
1454                                 condlog(2, "%s: Disable queueing", mpp->alias);
1455                         }
1456                 }
1457         }
1458 }
1459
1460 int update_prio(struct path *pp, int refresh_all)
1461 {
1462         int oldpriority;
1463         struct path *pp1;
1464         struct pathgroup * pgp;
1465         int i, j, changed = 0;
1466         struct config *conf;
1467
1468         if (refresh_all) {
1469                 vector_foreach_slot (pp->mpp->pg, pgp, i) {
1470                         vector_foreach_slot (pgp->paths, pp1, j) {
1471                                 oldpriority = pp1->priority;
1472                                 conf = get_multipath_config();
1473                                 pathinfo(pp1, conf, DI_PRIO);
1474                                 put_multipath_config(conf);
1475                                 if (pp1->priority != oldpriority)
1476                                         changed = 1;
1477                         }
1478                 }
1479                 return changed;
1480         }
1481         oldpriority = pp->priority;
1482         conf = get_multipath_config();
1483         if (pp->state != PATH_DOWN)
1484                 pathinfo(pp, conf, DI_PRIO);
1485         put_multipath_config(conf);
1486
1487         if (pp->priority == oldpriority)
1488                 return 0;
1489         return 1;
1490 }
1491
1492 int update_path_groups(struct multipath *mpp, struct vectors *vecs, int refresh)
1493 {
1494         if (reload_map(vecs, mpp, refresh, 1))
1495                 return 1;
1496
1497         dm_lib_release();
1498         if (setup_multipath(vecs, mpp) != 0)
1499                 return 1;
1500         sync_map_state(mpp);
1501
1502         return 0;
1503 }
1504
1505 void repair_path(struct path * pp)
1506 {
1507         if (pp->state != PATH_DOWN)
1508                 return;
1509
1510         checker_repair(&pp->checker);
1511         LOG_MSG(1, checker_message(&pp->checker));
1512 }
1513
1514 /*
1515  * Returns '1' if the path has been checked, '-1' if it was blacklisted
1516  * and '0' otherwise
1517  */
1518 int
1519 check_path (struct vectors * vecs, struct path * pp, int ticks)
1520 {
1521         int newstate;
1522         int new_path_up = 0;
1523         int chkr_new_path_up = 0;
1524         int add_active;
1525         int disable_reinstate = 0;
1526         int oldchkrstate = pp->chkrstate;
1527         int retrigger_tries, checkint;
1528         struct config *conf;
1529         int ret;
1530
1531         if ((pp->initialized == INIT_OK ||
1532              pp->initialized == INIT_REQUESTED_UDEV) && !pp->mpp)
1533                 return 0;
1534
1535         if (pp->tick)
1536                 pp->tick -= (pp->tick > ticks) ? ticks : pp->tick;
1537         if (pp->tick)
1538                 return 0; /* don't check this path yet */
1539
1540         conf = get_multipath_config();
1541         retrigger_tries = conf->retrigger_tries;
1542         checkint = conf->checkint;
1543         put_multipath_config(conf);
1544         if (!pp->mpp && pp->initialized == INIT_MISSING_UDEV &&
1545             pp->retriggers < retrigger_tries) {
1546                 condlog(2, "%s: triggering change event to reinitialize",
1547                         pp->dev);
1548                 pp->initialized = INIT_REQUESTED_UDEV;
1549                 pp->retriggers++;
1550                 sysfs_attr_set_value(pp->udev, "uevent", "change",
1551                                      strlen("change"));
1552                 return 0;
1553         }
1554
1555         /*
1556          * provision a next check soonest,
1557          * in case we exit abnormaly from here
1558          */
1559         pp->tick = checkint;
1560
1561         newstate = path_offline(pp);
1562         /*
1563          * Wait for uevent for removed paths;
1564          * some LLDDs like zfcp keep paths unavailable
1565          * without sending uevents.
1566          */
1567         if (newstate == PATH_REMOVED)
1568                 newstate = PATH_DOWN;
1569
1570         if (newstate == PATH_UP) {
1571                 conf = get_multipath_config();
1572                 newstate = get_state(pp, conf, 1, newstate);
1573                 put_multipath_config(conf);
1574         } else
1575                 checker_clear_message(&pp->checker);
1576
1577         if (pp->wwid_changed) {
1578                 condlog(2, "%s: path wwid has changed. Refusing to use",
1579                         pp->dev);
1580                 newstate = PATH_DOWN;
1581         }
1582
1583         if (newstate == PATH_WILD || newstate == PATH_UNCHECKED) {
1584                 condlog(2, "%s: unusable path", pp->dev);
1585                 conf = get_multipath_config();
1586                 pathinfo(pp, conf, 0);
1587                 put_multipath_config(conf);
1588                 return 1;
1589         }
1590         if (!pp->mpp) {
1591                 if (!strlen(pp->wwid) && pp->initialized != INIT_MISSING_UDEV &&
1592                     (newstate == PATH_UP || newstate == PATH_GHOST)) {
1593                         condlog(2, "%s: add missing path", pp->dev);
1594                         conf = get_multipath_config();
1595                         ret = pathinfo(pp, conf, DI_ALL | DI_BLACKLIST);
1596                         if (ret == PATHINFO_OK) {
1597                                 ev_add_path(pp, vecs, 1);
1598                                 pp->tick = 1;
1599                         } else if (ret == PATHINFO_SKIPPED) {
1600                                 put_multipath_config(conf);
1601                                 return -1;
1602                         }
1603                         put_multipath_config(conf);
1604                 }
1605                 return 0;
1606         }
1607         /*
1608          * Async IO in flight. Keep the previous path state
1609          * and reschedule as soon as possible
1610          */
1611         if (newstate == PATH_PENDING) {
1612                 pp->tick = 1;
1613                 return 0;
1614         }
1615         /*
1616          * Synchronize with kernel state
1617          */
1618         if (update_multipath_strings(pp->mpp, vecs->pathvec, 1)) {
1619                 condlog(1, "%s: Could not synchronize with kernel state",
1620                         pp->dev);
1621                 pp->dmstate = PSTATE_UNDEF;
1622         }
1623         /* if update_multipath_strings orphaned the path, quit early */
1624         if (!pp->mpp)
1625                 return 0;
1626
1627         if (pp->io_err_disable_reinstate && hit_io_err_recheck_time(pp)) {
1628                 pp->state = PATH_SHAKY;
1629                 /*
1630                  * to reschedule as soon as possible,so that this path can
1631                  * be recoverd in time
1632                  */
1633                 pp->tick = 1;
1634                 return 1;
1635         }
1636
1637         if ((newstate == PATH_UP || newstate == PATH_GHOST) &&
1638              pp->wait_checks > 0) {
1639                 if (pp->mpp->nr_active > 0) {
1640                         pp->state = PATH_DELAYED;
1641                         pp->wait_checks--;
1642                         return 1;
1643                 } else
1644                         pp->wait_checks = 0;
1645         }
1646
1647         /*
1648          * don't reinstate failed path, if its in stand-by
1649          * and if target supports only implicit tpgs mode.
1650          * this will prevent unnecessary i/o by dm on stand-by
1651          * paths if there are no other active paths in map.
1652          */
1653         disable_reinstate = (newstate == PATH_GHOST &&
1654                             pp->mpp->nr_active == 0 &&
1655                             pp->tpgs == TPGS_IMPLICIT) ? 1 : 0;
1656
1657         pp->chkrstate = newstate;
1658         if (newstate != pp->state) {
1659                 int oldstate = pp->state;
1660                 pp->state = newstate;
1661
1662                 LOG_MSG(1, checker_message(&pp->checker));
1663
1664                 /*
1665                  * upon state change, reset the checkint
1666                  * to the shortest delay
1667                  */
1668                 conf = get_multipath_config();
1669                 pp->checkint = conf->checkint;
1670                 put_multipath_config(conf);
1671
1672                 if (newstate != PATH_UP && newstate != PATH_GHOST) {
1673                         /*
1674                          * proactively fail path in the DM
1675                          */
1676                         if (oldstate == PATH_UP ||
1677                             oldstate == PATH_GHOST) {
1678                                 fail_path(pp, 1);
1679                                 if (pp->mpp->delay_wait_checks > 0 &&
1680                                     pp->watch_checks > 0) {
1681                                         pp->wait_checks = pp->mpp->delay_wait_checks;
1682                                         pp->watch_checks = 0;
1683                                 }
1684                         }else
1685                                 fail_path(pp, 0);
1686
1687                         /*
1688                          * cancel scheduled failback
1689                          */
1690                         pp->mpp->failback_tick = 0;
1691
1692                         pp->mpp->stat_path_failures++;
1693                         repair_path(pp);
1694                         return 1;
1695                 }
1696
1697                 if(newstate == PATH_UP || newstate == PATH_GHOST){
1698                         if ( pp->mpp && pp->mpp->prflag ){
1699                                 /*
1700                                  * Check Persistent Reservation.
1701                                  */
1702                         condlog(2, "%s: checking persistent reservation "
1703                                 "registration", pp->dev);
1704                         mpath_pr_event_handle(pp);
1705                         }
1706                 }
1707
1708                 /*
1709                  * reinstate this path
1710                  */
1711                 if (oldstate != PATH_UP &&
1712                     oldstate != PATH_GHOST) {
1713                         if (pp->mpp->delay_watch_checks > 0)
1714                                 pp->watch_checks = pp->mpp->delay_watch_checks;
1715                         add_active = 1;
1716                 } else {
1717                         if (pp->watch_checks > 0)
1718                                 pp->watch_checks--;
1719                         add_active = 0;
1720                 }
1721                 if (!disable_reinstate && reinstate_path(pp, add_active)) {
1722                         condlog(3, "%s: reload map", pp->dev);
1723                         ev_add_path(pp, vecs, 1);
1724                         pp->tick = 1;
1725                         return 0;
1726                 }
1727                 new_path_up = 1;
1728
1729                 if (oldchkrstate != PATH_UP && oldchkrstate != PATH_GHOST)
1730                         chkr_new_path_up = 1;
1731
1732                 /*
1733                  * if at least one path is up in a group, and
1734                  * the group is disabled, re-enable it
1735                  */
1736                 if (newstate == PATH_UP)
1737                         enable_group(pp);
1738         }
1739         else if (newstate == PATH_UP || newstate == PATH_GHOST) {
1740                 if ((pp->dmstate == PSTATE_FAILED ||
1741                     pp->dmstate == PSTATE_UNDEF) &&
1742                     !disable_reinstate) {
1743                         /* Clear IO errors */
1744                         if (reinstate_path(pp, 0)) {
1745                                 condlog(3, "%s: reload map", pp->dev);
1746                                 ev_add_path(pp, vecs, 1);
1747                                 pp->tick = 1;
1748                                 return 0;
1749                         }
1750                 } else {
1751                         unsigned int max_checkint;
1752                         LOG_MSG(4, checker_message(&pp->checker));
1753                         conf = get_multipath_config();
1754                         max_checkint = conf->max_checkint;
1755                         put_multipath_config(conf);
1756                         if (pp->checkint != max_checkint) {
1757                                 /*
1758                                  * double the next check delay.
1759                                  * max at conf->max_checkint
1760                                  */
1761                                 if (pp->checkint < (max_checkint / 2))
1762                                         pp->checkint = 2 * pp->checkint;
1763                                 else
1764                                         pp->checkint = max_checkint;
1765
1766                                 condlog(4, "%s: delay next check %is",
1767                                         pp->dev_t, pp->checkint);
1768                         }
1769                         if (pp->watch_checks > 0)
1770                                 pp->watch_checks--;
1771                         pp->tick = pp->checkint;
1772                 }
1773         }
1774         else if (newstate != PATH_UP && newstate != PATH_GHOST) {
1775                 if (pp->dmstate == PSTATE_ACTIVE ||
1776                     pp->dmstate == PSTATE_UNDEF)
1777                         fail_path(pp, 0);
1778                 if (newstate == PATH_DOWN) {
1779                         int log_checker_err;
1780
1781                         conf = get_multipath_config();
1782                         log_checker_err = conf->log_checker_err;
1783                         put_multipath_config(conf);
1784                         if (log_checker_err == LOG_CHKR_ERR_ONCE)
1785                                 LOG_MSG(3, checker_message(&pp->checker));
1786                         else
1787                                 LOG_MSG(2, checker_message(&pp->checker));
1788                 }
1789         }
1790
1791         pp->state = newstate;
1792         repair_path(pp);
1793
1794         if (pp->mpp->wait_for_udev)
1795                 return 1;
1796         /*
1797          * path prio refreshing
1798          */
1799         condlog(4, "path prio refresh");
1800
1801         if (update_prio(pp, new_path_up) &&
1802             (pp->mpp->pgpolicyfn == (pgpolicyfn *)group_by_prio) &&
1803              pp->mpp->pgfailback == -FAILBACK_IMMEDIATE)
1804                 update_path_groups(pp->mpp, vecs, !new_path_up);
1805         else if (need_switch_pathgroup(pp->mpp, 0)) {
1806                 if (pp->mpp->pgfailback > 0 &&
1807                     (new_path_up || pp->mpp->failback_tick <= 0))
1808                         pp->mpp->failback_tick =
1809                                 pp->mpp->pgfailback + 1;
1810                 else if (pp->mpp->pgfailback == -FAILBACK_IMMEDIATE ||
1811                          (chkr_new_path_up && followover_should_failback(pp)))
1812                         switch_pathgroup(pp->mpp);
1813         }
1814         return 1;
1815 }
1816
1817 static void init_path_check_interval(struct vectors *vecs)
1818 {
1819         struct config *conf;
1820         struct path *pp;
1821         unsigned int i;
1822
1823         vector_foreach_slot (vecs->pathvec, pp, i) {
1824                 conf = get_multipath_config();
1825                 pp->checkint = conf->checkint;
1826                 put_multipath_config(conf);
1827         }
1828 }
1829
1830 static void *
1831 checkerloop (void *ap)
1832 {
1833         struct vectors *vecs;
1834         struct path *pp;
1835         int count = 0;
1836         unsigned int i;
1837         struct itimerval timer_tick_it;
1838         struct timespec last_time;
1839         struct config *conf;
1840
1841         pthread_cleanup_push(rcu_unregister, NULL);
1842         rcu_register_thread();
1843         mlockall(MCL_CURRENT | MCL_FUTURE);
1844         vecs = (struct vectors *)ap;
1845         condlog(2, "path checkers start up");
1846
1847         /* Tweak start time for initial path check */
1848         if (clock_gettime(CLOCK_MONOTONIC, &last_time) != 0)
1849                 last_time.tv_sec = 0;
1850         else
1851                 last_time.tv_sec -= 1;
1852
1853         while (1) {
1854                 struct timespec diff_time, start_time, end_time;
1855                 int num_paths = 0, ticks = 0, signo, strict_timing, rc = 0;
1856                 sigset_t mask;
1857
1858                 if (clock_gettime(CLOCK_MONOTONIC, &start_time) != 0)
1859                         start_time.tv_sec = 0;
1860                 if (start_time.tv_sec && last_time.tv_sec) {
1861                         timespecsub(&start_time, &last_time, &diff_time);
1862                         condlog(4, "tick (%lu.%06lu secs)",
1863                                 diff_time.tv_sec, diff_time.tv_nsec / 1000);
1864                         last_time = start_time;
1865                         ticks = diff_time.tv_sec;
1866                 } else {
1867                         ticks = 1;
1868                         condlog(4, "tick (%d ticks)", ticks);
1869                 }
1870 #ifdef USE_SYSTEMD
1871                 if (use_watchdog)
1872                         sd_notify(0, "WATCHDOG=1");
1873 #endif
1874                 rc = set_config_state(DAEMON_RUNNING);
1875                 if (rc == ETIMEDOUT) {
1876                         condlog(4, "timeout waiting for DAEMON_IDLE");
1877                         continue;
1878                 }
1879
1880                 pthread_cleanup_push(cleanup_lock, &vecs->lock);
1881                 lock(&vecs->lock);
1882                 pthread_testcancel();
1883                 vector_foreach_slot (vecs->pathvec, pp, i) {
1884                         rc = check_path(vecs, pp, ticks);
1885                         if (rc < 0) {
1886                                 vector_del_slot(vecs->pathvec, i);
1887                                 free_path(pp);
1888                                 i--;
1889                         } else
1890                                 num_paths += rc;
1891                 }
1892                 lock_cleanup_pop(vecs->lock);
1893
1894                 pthread_cleanup_push(cleanup_lock, &vecs->lock);
1895                 lock(&vecs->lock);
1896                 pthread_testcancel();
1897                 defered_failback_tick(vecs->mpvec);
1898                 retry_count_tick(vecs->mpvec);
1899                 missing_uev_wait_tick(vecs);
1900                 ghost_delay_tick(vecs);
1901                 lock_cleanup_pop(vecs->lock);
1902
1903                 if (count)
1904                         count--;
1905                 else {
1906                         pthread_cleanup_push(cleanup_lock, &vecs->lock);
1907                         lock(&vecs->lock);
1908                         pthread_testcancel();
1909                         condlog(4, "map garbage collection");
1910                         mpvec_garbage_collector(vecs);
1911                         count = MAPGCINT;
1912                         lock_cleanup_pop(vecs->lock);
1913                 }
1914
1915                 diff_time.tv_nsec = 0;
1916                 if (start_time.tv_sec &&
1917                     clock_gettime(CLOCK_MONOTONIC, &end_time) == 0) {
1918                         timespecsub(&end_time, &start_time, &diff_time);
1919                         if (num_paths) {
1920                                 unsigned int max_checkint;
1921
1922                                 condlog(3, "checked %d path%s in %lu.%06lu secs",
1923                                         num_paths, num_paths > 1 ? "s" : "",
1924                                         diff_time.tv_sec,
1925                                         diff_time.tv_nsec / 1000);
1926                                 conf = get_multipath_config();
1927                                 max_checkint = conf->max_checkint;
1928                                 put_multipath_config(conf);
1929                                 if (diff_time.tv_sec > max_checkint)
1930                                         condlog(1, "path checkers took longer "
1931                                                 "than %lu seconds, consider "
1932                                                 "increasing max_polling_interval",
1933                                                 diff_time.tv_sec);
1934                         }
1935                 }
1936
1937                 post_config_state(DAEMON_IDLE);
1938                 conf = get_multipath_config();
1939                 strict_timing = conf->strict_timing;
1940                 put_multipath_config(conf);
1941                 if (!strict_timing)
1942                         sleep(1);
1943                 else {
1944                         timer_tick_it.it_interval.tv_sec = 0;
1945                         timer_tick_it.it_interval.tv_usec = 0;
1946                         if (diff_time.tv_nsec) {
1947                                 timer_tick_it.it_value.tv_sec = 0;
1948                                 timer_tick_it.it_value.tv_usec =
1949                                      1000UL * 1000 * 1000 - diff_time.tv_nsec;
1950                         } else {
1951                                 timer_tick_it.it_value.tv_sec = 1;
1952                                 timer_tick_it.it_value.tv_usec = 0;
1953                         }
1954                         setitimer(ITIMER_REAL, &timer_tick_it, NULL);
1955
1956                         sigemptyset(&mask);
1957                         sigaddset(&mask, SIGALRM);
1958                         condlog(3, "waiting for %lu.%06lu secs",
1959                                 timer_tick_it.it_value.tv_sec,
1960                                 timer_tick_it.it_value.tv_usec);
1961                         if (sigwait(&mask, &signo) != 0) {
1962                                 condlog(3, "sigwait failed with error %d",
1963                                         errno);
1964                                 conf = get_multipath_config();
1965                                 conf->strict_timing = 0;
1966                                 put_multipath_config(conf);
1967                                 break;
1968                         }
1969                 }
1970         }
1971         pthread_cleanup_pop(1);
1972         return NULL;
1973 }
1974
1975 int
1976 configure (struct vectors * vecs, int start_waiters)
1977 {
1978         struct multipath * mpp;
1979         struct path * pp;
1980         vector mpvec;
1981         int i, ret;
1982         struct config *conf;
1983         static int force_reload = FORCE_RELOAD_WEAK;
1984
1985         if (!vecs->pathvec && !(vecs->pathvec = vector_alloc())) {
1986                 condlog(0, "couldn't allocate path vec in configure");
1987                 return 1;
1988         }
1989
1990         if (!vecs->mpvec && !(vecs->mpvec = vector_alloc())) {
1991                 condlog(0, "couldn't allocate multipath vec in configure");
1992                 return 1;
1993         }
1994
1995         if (!(mpvec = vector_alloc())) {
1996                 condlog(0, "couldn't allocate new maps vec in configure");
1997                 return 1;
1998         }
1999
2000         /*
2001          * probe for current path (from sysfs) and map (from dm) sets
2002          */
2003         ret = path_discovery(vecs->pathvec, DI_ALL);
2004         if (ret < 0) {
2005                 condlog(0, "configure failed at path discovery");
2006                 return 1;
2007         }
2008
2009         vector_foreach_slot (vecs->pathvec, pp, i){
2010                 conf = get_multipath_config();
2011                 if (filter_path(conf, pp) > 0){
2012                         vector_del_slot(vecs->pathvec, i);
2013                         free_path(pp);
2014                         i--;
2015                 }
2016                 else
2017                         pp->checkint = conf->checkint;
2018                 put_multipath_config(conf);
2019         }
2020         if (map_discovery(vecs)) {
2021                 condlog(0, "configure failed at map discovery");
2022                 return 1;
2023         }
2024
2025         /*
2026          * create new set of maps & push changed ones into dm
2027          * In the first call, use FORCE_RELOAD_WEAK to avoid making
2028          * superfluous ACT_RELOAD ioctls. Later calls are done
2029          * with FORCE_RELOAD_YES.
2030          */
2031         ret = coalesce_paths(vecs, mpvec, NULL, force_reload, CMD_NONE);
2032         if (force_reload == FORCE_RELOAD_WEAK)
2033                 force_reload = FORCE_RELOAD_YES;
2034         if (ret) {
2035                 condlog(0, "configure failed while coalescing paths");
2036                 return 1;
2037         }
2038
2039         /*
2040          * may need to remove some maps which are no longer relevant
2041          * e.g., due to blacklist changes in conf file
2042          */
2043         if (coalesce_maps(vecs, mpvec)) {
2044                 condlog(0, "configure failed while coalescing maps");
2045                 return 1;
2046         }
2047
2048         dm_lib_release();
2049
2050         sync_maps_state(mpvec);
2051         vector_foreach_slot(mpvec, mpp, i){
2052                 remember_wwid(mpp->wwid);
2053                 update_map_pr(mpp);
2054         }
2055
2056         /*
2057          * purge dm of old maps
2058          */
2059         remove_maps(vecs);
2060
2061         /*
2062          * save new set of maps formed by considering current path state
2063          */
2064         vector_free(vecs->mpvec);
2065         vecs->mpvec = mpvec;
2066
2067         /*
2068          * start dm event waiter threads for these new maps
2069          */
2070         vector_foreach_slot(vecs->mpvec, mpp, i) {
2071                 if (setup_multipath(vecs, mpp)) {
2072                         i--;
2073                         continue;
2074                 }
2075                 if (start_waiters) {
2076                         if (start_waiter_thread(mpp, vecs)) {
2077                                 remove_map(mpp, vecs, 1);
2078                                 i--;
2079                         }
2080                 }
2081         }
2082         return 0;
2083 }
2084
2085 int
2086 need_to_delay_reconfig(struct vectors * vecs)
2087 {
2088         struct multipath *mpp;
2089         int i;
2090
2091         if (!VECTOR_SIZE(vecs->mpvec))
2092                 return 0;
2093
2094         vector_foreach_slot(vecs->mpvec, mpp, i) {
2095                 if (mpp->wait_for_udev)
2096                         return 1;
2097         }
2098         return 0;
2099 }
2100
2101 void rcu_free_config(struct rcu_head *head)
2102 {
2103         struct config *conf = container_of(head, struct config, rcu);
2104
2105         free_config(conf);
2106 }
2107
2108 int
2109 reconfigure (struct vectors * vecs)
2110 {
2111         struct config * old, *conf;
2112
2113         conf = load_config(DEFAULT_CONFIGFILE);
2114         if (!conf)
2115                 return 1;
2116
2117         /*
2118          * free old map and path vectors ... they use old conf state
2119          */
2120         if (VECTOR_SIZE(vecs->mpvec))
2121                 remove_maps_and_stop_waiters(vecs);
2122
2123         free_pathvec(vecs->pathvec, FREE_PATHS);
2124         vecs->pathvec = NULL;
2125
2126         /* Re-read any timezone changes */
2127         tzset();
2128
2129         dm_drv_version(conf->version, TGT_MPATH);
2130         if (verbosity)
2131                 conf->verbosity = verbosity;
2132         if (bindings_read_only)
2133                 conf->bindings_read_only = bindings_read_only;
2134         if (conf->find_multipaths) {
2135                 condlog(2, "find_multipaths is set: -n is implied");
2136                 ignore_new_devs = 1;
2137         }
2138         if (ignore_new_devs)
2139                 conf->ignore_new_devs = ignore_new_devs;
2140         uxsock_timeout = conf->uxsock_timeout;
2141
2142         old = rcu_dereference(multipath_conf);
2143         rcu_assign_pointer(multipath_conf, conf);
2144         call_rcu(&old->rcu, rcu_free_config);
2145
2146         configure(vecs, 1);
2147
2148
2149         return 0;
2150 }
2151
2152 static struct vectors *
2153 init_vecs (void)
2154 {
2155         struct vectors * vecs;
2156
2157         vecs = (struct vectors *)MALLOC(sizeof(struct vectors));
2158
2159         if (!vecs)
2160                 return NULL;
2161
2162         pthread_mutex_init(&vecs->lock.mutex, NULL);
2163
2164         return vecs;
2165 }
2166
2167 static void *
2168 signal_set(int signo, void (*func) (int))
2169 {
2170         int r;
2171         struct sigaction sig;
2172         struct sigaction osig;
2173
2174         sig.sa_handler = func;
2175         sigemptyset(&sig.sa_mask);
2176         sig.sa_flags = 0;
2177
2178         r = sigaction(signo, &sig, &osig);
2179
2180         if (r < 0)
2181                 return (SIG_ERR);
2182         else
2183                 return (osig.sa_handler);
2184 }
2185
2186 void
2187 handle_signals(void)
2188 {
2189         if (exit_sig) {
2190                 condlog(2, "exit (signal)");
2191                 exit_daemon();
2192         }
2193         if (reconfig_sig) {
2194                 condlog(2, "reconfigure (signal)");
2195                 set_config_state(DAEMON_CONFIGURE);
2196         }
2197         if (log_reset_sig) {
2198                 condlog(2, "reset log (signal)");
2199                 pthread_mutex_lock(&logq_lock);
2200                 log_reset("multipathd");
2201                 pthread_mutex_unlock(&logq_lock);
2202         }
2203         exit_sig = 0;
2204         reconfig_sig = 0;
2205         log_reset_sig = 0;
2206 }
2207
2208 static void
2209 sighup (int sig)
2210 {
2211         reconfig_sig = 1;
2212 }
2213
2214 static void
2215 sigend (int sig)
2216 {
2217         exit_sig = 1;
2218 }
2219
2220 static void
2221 sigusr1 (int sig)
2222 {
2223         log_reset_sig = 1;
2224 }
2225
2226 static void
2227 sigusr2 (int sig)
2228 {
2229         condlog(3, "SIGUSR2 received");
2230 }
2231
2232 static void
2233 signal_init(void)
2234 {
2235         sigset_t set;
2236
2237         sigemptyset(&set);
2238         sigaddset(&set, SIGUSR2);
2239         pthread_sigmask(SIG_SETMASK, &set, NULL);
2240
2241         signal_set(SIGHUP, sighup);
2242         signal_set(SIGUSR1, sigusr1);
2243         signal_set(SIGUSR2, sigusr2);
2244         signal_set(SIGINT, sigend);
2245         signal_set(SIGTERM, sigend);
2246         signal_set(SIGPIPE, sigend);
2247 }
2248
2249 static void
2250 setscheduler (void)
2251 {
2252         int res;
2253         static struct sched_param sched_param = {
2254                 .sched_priority = 99
2255         };
2256
2257         res = sched_setscheduler (0, SCHED_RR, &sched_param);
2258
2259         if (res == -1)
2260                 condlog(LOG_WARNING, "Could not set SCHED_RR at priority 99");
2261         return;
2262 }
2263
2264 static void
2265 set_oom_adj (void)
2266 {
2267 #ifdef OOM_SCORE_ADJ_MIN
2268         int retry = 1;
2269         char *file = "/proc/self/oom_score_adj";
2270         int score = OOM_SCORE_ADJ_MIN;
2271 #else
2272         int retry = 0;
2273         char *file = "/proc/self/oom_adj";
2274         int score = OOM_ADJUST_MIN;
2275 #endif
2276         FILE *fp;
2277         struct stat st;
2278         char *envp;
2279
2280         envp = getenv("OOMScoreAdjust");
2281         if (envp) {
2282                 condlog(3, "Using systemd provided OOMScoreAdjust");
2283                 return;
2284         }
2285         do {
2286                 if (stat(file, &st) == 0){
2287                         fp = fopen(file, "w");
2288                         if (!fp) {
2289                                 condlog(0, "couldn't fopen %s : %s", file,
2290                                         strerror(errno));
2291                                 return;
2292                         }
2293                         fprintf(fp, "%i", score);
2294                         fclose(fp);
2295                         return;
2296                 }
2297                 if (errno != ENOENT) {
2298                         condlog(0, "couldn't stat %s : %s", file,
2299                                 strerror(errno));
2300                         return;
2301                 }
2302 #ifdef OOM_ADJUST_MIN
2303                 file = "/proc/self/oom_adj";
2304                 score = OOM_ADJUST_MIN;
2305 #else
2306                 retry = 0;
2307 #endif
2308         } while (retry--);
2309         condlog(0, "couldn't adjust oom score");
2310 }
2311
2312 static int
2313 child (void * param)
2314 {
2315         pthread_t check_thr, uevent_thr, uxlsnr_thr, uevq_thr;
2316         pthread_attr_t log_attr, misc_attr, uevent_attr;
2317         struct vectors * vecs;
2318         struct multipath * mpp;
2319         int i;
2320 #ifdef USE_SYSTEMD
2321         unsigned long checkint;
2322         int startup_done = 0;
2323 #endif
2324         int rc;
2325         int pid_fd = -1;
2326         struct config *conf;
2327         char *envp;
2328
2329         mlockall(MCL_CURRENT | MCL_FUTURE);
2330         signal_init();
2331         rcu_init();
2332
2333         setup_thread_attr(&misc_attr, 64 * 1024, 0);
2334         setup_thread_attr(&uevent_attr, DEFAULT_UEVENT_STACKSIZE * 1024, 0);
2335         setup_thread_attr(&waiter_attr, 32 * 1024, 1);
2336         setup_thread_attr(&io_err_stat_attr, 32 * 1024, 0);
2337
2338         if (logsink == 1) {
2339                 setup_thread_attr(&log_attr, 64 * 1024, 0);
2340                 log_thread_start(&log_attr);
2341                 pthread_attr_destroy(&log_attr);
2342         }
2343         pid_fd = pidfile_create(DEFAULT_PIDFILE, daemon_pid);
2344         if (pid_fd < 0) {
2345                 condlog(1, "failed to create pidfile");
2346                 if (logsink == 1)
2347                         log_thread_stop();
2348                 exit(1);
2349         }
2350
2351         post_config_state(DAEMON_START);
2352
2353         condlog(2, "--------start up--------");
2354         condlog(2, "read " DEFAULT_CONFIGFILE);
2355
2356         conf = load_config(DEFAULT_CONFIGFILE);
2357         if (!conf)
2358                 goto failed;
2359
2360         if (verbosity)
2361                 conf->verbosity = verbosity;
2362         if (bindings_read_only)
2363                 conf->bindings_read_only = bindings_read_only;
2364         if (ignore_new_devs)
2365                 conf->ignore_new_devs = ignore_new_devs;
2366         uxsock_timeout = conf->uxsock_timeout;
2367         rcu_assign_pointer(multipath_conf, conf);
2368         if (init_checkers(conf->multipath_dir)) {
2369                 condlog(0, "failed to initialize checkers");
2370                 goto failed;
2371         }
2372         if (init_prio(conf->multipath_dir)) {
2373                 condlog(0, "failed to initialize prioritizers");
2374                 goto failed;
2375         }
2376
2377         setlogmask(LOG_UPTO(conf->verbosity + 3));
2378
2379         envp = getenv("LimitNOFILE");
2380
2381         if (envp) {
2382                 condlog(2,"Using systemd provided open fds limit of %s", envp);
2383         } else if (conf->max_fds) {
2384                 struct rlimit fd_limit;
2385
2386                 if (getrlimit(RLIMIT_NOFILE, &fd_limit) < 0) {
2387                         condlog(0, "can't get open fds limit: %s",
2388                                 strerror(errno));
2389                         fd_limit.rlim_cur = 0;
2390                         fd_limit.rlim_max = 0;
2391                 }
2392                 if (fd_limit.rlim_cur < conf->max_fds) {
2393                         fd_limit.rlim_cur = conf->max_fds;
2394                         if (fd_limit.rlim_max < conf->max_fds)
2395                                 fd_limit.rlim_max = conf->max_fds;
2396                         if (setrlimit(RLIMIT_NOFILE, &fd_limit) < 0) {
2397                                 condlog(0, "can't set open fds limit to "
2398                                         "%lu/%lu : %s",
2399                                         fd_limit.rlim_cur, fd_limit.rlim_max,
2400                                         strerror(errno));
2401                         } else {
2402                                 condlog(3, "set open fds limit to %lu/%lu",
2403                                         fd_limit.rlim_cur, fd_limit.rlim_max);
2404                         }
2405                 }
2406
2407         }
2408
2409         vecs = gvecs = init_vecs();
2410         if (!vecs)
2411                 goto failed;
2412
2413         setscheduler();
2414         set_oom_adj();
2415
2416 #ifdef USE_SYSTEMD
2417         envp = getenv("WATCHDOG_USEC");
2418         if (envp && sscanf(envp, "%lu", &checkint) == 1) {
2419                 /* Value is in microseconds */
2420                 conf->max_checkint = checkint / 1000000;
2421                 /* Rescale checkint */
2422                 if (conf->checkint > conf->max_checkint)
2423                         conf->checkint = conf->max_checkint;
2424                 else
2425                         conf->checkint = conf->max_checkint / 4;
2426                 condlog(3, "enabling watchdog, interval %d max %d",
2427                         conf->checkint, conf->max_checkint);
2428                 use_watchdog = conf->checkint;
2429         }
2430 #endif
2431         /*
2432          * Startup done, invalidate configuration
2433          */
2434         conf = NULL;
2435
2436         /*
2437          * Signal start of configuration
2438          */
2439         post_config_state(DAEMON_CONFIGURE);
2440
2441         init_path_check_interval(vecs);
2442
2443         /*
2444          * Start uevent listener early to catch events
2445          */
2446         if ((rc = pthread_create(&uevent_thr, &uevent_attr, ueventloop, udev))) {
2447                 condlog(0, "failed to create uevent thread: %d", rc);
2448                 goto failed;
2449         }
2450         pthread_attr_destroy(&uevent_attr);
2451         if ((rc = pthread_create(&uxlsnr_thr, &misc_attr, uxlsnrloop, vecs))) {
2452                 condlog(0, "failed to create cli listener: %d", rc);
2453                 goto failed;
2454         }
2455
2456         /*
2457          * start threads
2458          */
2459         rc = start_io_err_stat_thread(vecs);
2460         if (rc)
2461                 goto failed;
2462
2463         if ((rc = pthread_create(&check_thr, &misc_attr, checkerloop, vecs))) {
2464                 condlog(0,"failed to create checker loop thread: %d", rc);
2465                 goto failed;
2466         }
2467         if ((rc = pthread_create(&uevq_thr, &misc_attr, uevqloop, vecs))) {
2468                 condlog(0, "failed to create uevent dispatcher: %d", rc);
2469                 goto failed;
2470         }
2471         pthread_attr_destroy(&misc_attr);
2472
2473         while (running_state != DAEMON_SHUTDOWN) {
2474                 pthread_cleanup_push(config_cleanup, NULL);
2475                 pthread_mutex_lock(&config_lock);
2476                 if (running_state != DAEMON_CONFIGURE &&
2477                     running_state != DAEMON_SHUTDOWN) {
2478                         pthread_cond_wait(&config_cond, &config_lock);
2479                 }
2480                 pthread_cleanup_pop(1);
2481                 if (running_state == DAEMON_CONFIGURE) {
2482                         pthread_cleanup_push(cleanup_lock, &vecs->lock);
2483                         lock(&vecs->lock);
2484                         pthread_testcancel();
2485                         if (!need_to_delay_reconfig(vecs)) {
2486                                 reconfigure(vecs);
2487                         } else {
2488                                 conf = get_multipath_config();
2489                                 conf->delayed_reconfig = 1;
2490                                 put_multipath_config(conf);
2491                         }
2492                         lock_cleanup_pop(vecs->lock);
2493                         post_config_state(DAEMON_IDLE);
2494 #ifdef USE_SYSTEMD
2495                         if (!startup_done) {
2496                                 sd_notify(0, "READY=1");
2497                                 startup_done = 1;
2498                         }
2499 #endif
2500                 }
2501         }
2502
2503         lock(&vecs->lock);
2504         conf = get_multipath_config();
2505         if (conf->queue_without_daemon == QUE_NO_DAEMON_OFF)
2506                 vector_foreach_slot(vecs->mpvec, mpp, i)
2507                         dm_queue_if_no_path(mpp->alias, 0);
2508         put_multipath_config(conf);
2509         remove_maps_and_stop_waiters(vecs);
2510         unlock(&vecs->lock);
2511
2512         pthread_cancel(check_thr);
2513         pthread_cancel(uevent_thr);
2514         pthread_cancel(uxlsnr_thr);
2515         pthread_cancel(uevq_thr);
2516
2517         pthread_join(check_thr, NULL);
2518         pthread_join(uevent_thr, NULL);
2519         pthread_join(uxlsnr_thr, NULL);
2520         pthread_join(uevq_thr, NULL);
2521
2522         stop_io_err_stat_thread();
2523
2524         lock(&vecs->lock);
2525         free_pathvec(vecs->pathvec, FREE_PATHS);
2526         vecs->pathvec = NULL;
2527         unlock(&vecs->lock);
2528
2529         pthread_mutex_destroy(&vecs->lock.mutex);
2530         FREE(vecs);
2531         vecs = NULL;
2532
2533         cleanup_checkers();
2534         cleanup_prio();
2535
2536         dm_lib_release();
2537         dm_lib_exit();
2538
2539         /* We're done here */
2540         condlog(3, "unlink pidfile");
2541         unlink(DEFAULT_PIDFILE);
2542
2543         condlog(2, "--------shut down-------");
2544
2545         if (logsink == 1)
2546                 log_thread_stop();
2547
2548         /*
2549          * Freeing config must be done after condlog() and dm_lib_exit(),
2550          * because logging functions like dlog() and dm_write_log()
2551          * reference the config.
2552          */
2553         conf = rcu_dereference(multipath_conf);
2554         rcu_assign_pointer(multipath_conf, NULL);
2555         call_rcu(&conf->rcu, rcu_free_config);
2556         udev_unref(udev);
2557         udev = NULL;
2558         pthread_attr_destroy(&waiter_attr);
2559         pthread_attr_destroy(&io_err_stat_attr);
2560 #ifdef _DEBUG_
2561         dbg_free_final(NULL);
2562 #endif
2563
2564 #ifdef USE_SYSTEMD
2565         sd_notify(0, "ERRNO=0");
2566 #endif
2567         exit(0);
2568
2569 failed:
2570 #ifdef USE_SYSTEMD
2571         sd_notify(0, "ERRNO=1");
2572 #endif
2573         if (pid_fd >= 0)
2574                 close(pid_fd);
2575         exit(1);
2576 }
2577
2578 static int
2579 daemonize(void)
2580 {
2581         int pid;
2582         int dev_null_fd;
2583
2584         if( (pid = fork()) < 0){
2585                 fprintf(stderr, "Failed first fork : %s\n", strerror(errno));
2586                 return -1;
2587         }
2588         else if (pid != 0)
2589                 return pid;
2590
2591         setsid();
2592
2593         if ( (pid = fork()) < 0)
2594                 fprintf(stderr, "Failed second fork : %s\n", strerror(errno));
2595         else if (pid != 0)
2596                 _exit(0);
2597
2598         if (chdir("/") < 0)
2599                 fprintf(stderr, "cannot chdir to '/', continuing\n");
2600
2601         dev_null_fd = open("/dev/null", O_RDWR);
2602         if (dev_null_fd < 0){
2603                 fprintf(stderr, "cannot open /dev/null for input & output : %s\n",
2604                         strerror(errno));
2605                 _exit(0);
2606         }
2607
2608         close(STDIN_FILENO);
2609         if (dup(dev_null_fd) < 0) {
2610                 fprintf(stderr, "cannot dup /dev/null to stdin : %s\n",
2611                         strerror(errno));
2612                 _exit(0);
2613         }
2614         close(STDOUT_FILENO);
2615         if (dup(dev_null_fd) < 0) {
2616                 fprintf(stderr, "cannot dup /dev/null to stdout : %s\n",
2617                         strerror(errno));
2618                 _exit(0);
2619         }
2620         close(STDERR_FILENO);
2621         if (dup(dev_null_fd) < 0) {
2622                 fprintf(stderr, "cannot dup /dev/null to stderr : %s\n",
2623                         strerror(errno));
2624                 _exit(0);
2625         }
2626         close(dev_null_fd);
2627         daemon_pid = getpid();
2628         return 0;
2629 }
2630
2631 int
2632 main (int argc, char *argv[])
2633 {
2634         extern char *optarg;
2635         extern int optind;
2636         int arg;
2637         int err;
2638         int foreground = 0;
2639         struct config *conf;
2640
2641         ANNOTATE_BENIGN_RACE_SIZED(&multipath_conf, sizeof(multipath_conf),
2642                                    "Manipulated through RCU");
2643         ANNOTATE_BENIGN_RACE_SIZED(&running_state, sizeof(running_state),
2644                 "Suppress complaints about unprotected running_state reads");
2645         ANNOTATE_BENIGN_RACE_SIZED(&uxsock_timeout, sizeof(uxsock_timeout),
2646                 "Suppress complaints about this scalar variable");
2647
2648         logsink = 1;
2649
2650         if (getuid() != 0) {
2651                 fprintf(stderr, "need to be root\n");
2652                 exit(1);
2653         }
2654
2655         /* make sure we don't lock any path */
2656         if (chdir("/") < 0)
2657                 fprintf(stderr, "can't chdir to root directory : %s\n",
2658                         strerror(errno));
2659         umask(umask(077) | 022);
2660
2661         pthread_cond_init_mono(&config_cond);
2662
2663         udev = udev_new();
2664         libmp_udev_set_sync_support(0);
2665
2666         while ((arg = getopt(argc, argv, ":dsv:k::Bn")) != EOF ) {
2667                 switch(arg) {
2668                 case 'd':
2669                         foreground = 1;
2670                         if (logsink > 0)
2671                                 logsink = 0;
2672                         //debug=1; /* ### comment me out ### */
2673                         break;
2674                 case 'v':
2675                         if (sizeof(optarg) > sizeof(char *) ||
2676                             !isdigit(optarg[0]))
2677                                 exit(1);
2678
2679                         verbosity = atoi(optarg);
2680                         break;
2681                 case 's':
2682                         logsink = -1;
2683                         break;
2684                 case 'k':
2685                         conf = load_config(DEFAULT_CONFIGFILE);
2686                         if (!conf)
2687                                 exit(1);
2688                         if (verbosity)
2689                                 conf->verbosity = verbosity;
2690                         uxsock_timeout = conf->uxsock_timeout;
2691                         uxclnt(optarg, uxsock_timeout + 100);
2692                         free_config(conf);
2693                         exit(0);
2694                 case 'B':
2695                         bindings_read_only = 1;
2696                         break;
2697                 case 'n':
2698                         ignore_new_devs = 1;
2699                         break;
2700                 default:
2701                         fprintf(stderr, "Invalid argument '-%c'\n",
2702                                 optopt);
2703                         exit(1);
2704                 }
2705         }
2706         if (optind < argc) {
2707                 char cmd[CMDSIZE];
2708                 char * s = cmd;
2709                 char * c = s;
2710
2711                 conf = load_config(DEFAULT_CONFIGFILE);
2712                 if (!conf)
2713                         exit(1);
2714                 if (verbosity)
2715                         conf->verbosity = verbosity;
2716                 uxsock_timeout = conf->uxsock_timeout;
2717                 memset(cmd, 0x0, CMDSIZE);
2718                 while (optind < argc) {
2719                         if (strchr(argv[optind], ' '))
2720                                 c += snprintf(c, s + CMDSIZE - c, "\"%s\" ", argv[optind]);
2721                         else
2722                                 c += snprintf(c, s + CMDSIZE - c, "%s ", argv[optind]);
2723                         optind++;
2724                 }
2725                 c += snprintf(c, s + CMDSIZE - c, "\n");
2726                 uxclnt(s, uxsock_timeout + 100);
2727                 free_config(conf);
2728                 exit(0);
2729         }
2730
2731         if (foreground) {
2732                 if (!isatty(fileno(stdout)))
2733                         setbuf(stdout, NULL);
2734                 err = 0;
2735                 daemon_pid = getpid();
2736         } else
2737                 err = daemonize();
2738
2739         if (err < 0)
2740                 /* error */
2741                 exit(1);
2742         else if (err > 0)
2743                 /* parent dies */
2744                 exit(0);
2745         else
2746                 /* child lives */
2747                 return (child(NULL));
2748 }
2749
2750 void *  mpath_pr_event_handler_fn (void * pathp )
2751 {
2752         struct multipath * mpp;
2753         int i, ret, isFound;
2754         struct path * pp = (struct path *)pathp;
2755         struct prout_param_descriptor *param;
2756         struct prin_resp *resp;
2757
2758         mpp = pp->mpp;
2759
2760         resp = mpath_alloc_prin_response(MPATH_PRIN_RKEY_SA);
2761         if (!resp){
2762                 condlog(0,"%s Alloc failed for prin response", pp->dev);
2763                 return NULL;
2764         }
2765
2766         ret = prin_do_scsi_ioctl(pp->dev, MPATH_PRIN_RKEY_SA, resp, 0);
2767         if (ret != MPATH_PR_SUCCESS )
2768         {
2769                 condlog(0,"%s : pr in read keys service action failed. Error=%d", pp->dev, ret);
2770                 goto out;
2771         }
2772
2773         condlog(3, " event pr=%d addlen=%d",resp->prin_descriptor.prin_readkeys.prgeneration,
2774                         resp->prin_descriptor.prin_readkeys.additional_length );
2775
2776         if (resp->prin_descriptor.prin_readkeys.additional_length == 0 )
2777         {
2778                 condlog(1, "%s: No key found. Device may not be registered.", pp->dev);
2779                 ret = MPATH_PR_SUCCESS;
2780                 goto out;
2781         }
2782         condlog(2, "Multipath  reservation_key: 0x%" PRIx64 " ",
2783                 get_be64(mpp->reservation_key));
2784
2785         isFound =0;
2786         for (i = 0; i < resp->prin_descriptor.prin_readkeys.additional_length/8; i++ )
2787         {
2788                 condlog(2, "PR IN READKEYS[%d]  reservation key:",i);
2789                 dumpHex((char *)&resp->prin_descriptor.prin_readkeys.key_list[i*8], 8 , -1);
2790                 if (!memcmp(&mpp->reservation_key, &resp->prin_descriptor.prin_readkeys.key_list[i*8], 8))
2791                 {
2792                         condlog(2, "%s: pr key found in prin readkeys response", mpp->alias);
2793                         isFound =1;
2794                         break;
2795                 }
2796         }
2797         if (!isFound)
2798         {
2799                 condlog(0, "%s: Either device not registered or ", pp->dev);
2800                 condlog(0, "host is not authorised for registration. Skip path");
2801                 ret = MPATH_PR_OTHER;
2802                 goto out;
2803         }
2804
2805         param= malloc(sizeof(struct prout_param_descriptor));
2806         memset(param, 0 , sizeof(struct prout_param_descriptor));
2807         memcpy(param->sa_key, &mpp->reservation_key, 8);
2808         param->num_transportid = 0;
2809
2810         condlog(3, "device %s:%s", pp->dev, pp->mpp->wwid);
2811
2812         ret = prout_do_scsi_ioctl(pp->dev, MPATH_PROUT_REG_IGN_SA, 0, 0, param, 0);
2813         if (ret != MPATH_PR_SUCCESS )
2814         {
2815                 condlog(0,"%s: Reservation registration failed. Error: %d", pp->dev, ret);
2816         }
2817         mpp->prflag = 1;
2818
2819         free(param);
2820 out:
2821         free(resp);
2822         return NULL;
2823 }
2824
2825 int mpath_pr_event_handle(struct path *pp)
2826 {
2827         pthread_t thread;
2828         int rc;
2829         pthread_attr_t attr;
2830         struct multipath * mpp;
2831
2832         mpp = pp->mpp;
2833
2834         if (get_be64(mpp->reservation_key))
2835                 return -1;
2836
2837         pthread_attr_init(&attr);
2838         pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
2839
2840         rc = pthread_create(&thread, NULL , mpath_pr_event_handler_fn, pp);
2841         if (rc) {
2842                 condlog(0, "%s: ERROR; return code from pthread_create() is %d", pp->dev, rc);
2843                 return -1;
2844         }
2845         pthread_attr_destroy(&attr);
2846         rc = pthread_join(thread, NULL);
2847         return 0;
2848 }