c700a95d4e7854000ad86becba1193ab7d6ec7f2
[multipath-tools/.git] / multipathd / main.c
1 /*
2  * Copyright (c) 2004, 2005 Christophe Varoqui
3  * Copyright (c) 2005 Kiyoshi Ueda, NEC
4  * Copyright (c) 2005 Benjamin Marzinski, Redhat
5  * Copyright (c) 2005 Edward Goggin, EMC
6  */
7 #include <unistd.h>
8 #include <sys/stat.h>
9 #include <libdevmapper.h>
10 #include <sys/wait.h>
11 #include <sys/mman.h>
12 #include <sys/types.h>
13 #include <fcntl.h>
14 #include <errno.h>
15 #include <sys/time.h>
16 #include <sys/resource.h>
17 #include <limits.h>
18 #include <linux/oom.h>
19 #include <libudev.h>
20 #include <urcu.h>
21 #ifdef USE_SYSTEMD
22 #include <systemd/sd-daemon.h>
23 #endif
24 #include <semaphore.h>
25 #include <time.h>
26 #include <stdbool.h>
27
28 /*
29  * libmultipath
30  */
31 #include "time-util.h"
32
33 /*
34  * libcheckers
35  */
36 #include "checkers.h"
37
38 #ifdef USE_SYSTEMD
39 static int use_watchdog;
40 #endif
41
42 /*
43  * libmultipath
44  */
45 #include "parser.h"
46 #include "vector.h"
47 #include "memory.h"
48 #include "config.h"
49 #include "util.h"
50 #include "hwtable.h"
51 #include "defaults.h"
52 #include "structs.h"
53 #include "blacklist.h"
54 #include "structs_vec.h"
55 #include "dmparser.h"
56 #include "devmapper.h"
57 #include "sysfs.h"
58 #include "dict.h"
59 #include "discovery.h"
60 #include "debug.h"
61 #include "propsel.h"
62 #include "uevent.h"
63 #include "switchgroup.h"
64 #include "print.h"
65 #include "configure.h"
66 #include "prio.h"
67 #include "wwids.h"
68 #include "pgpolicies.h"
69 #include "uevent.h"
70 #include "log.h"
71
72 #include "mpath_cmd.h"
73 #include "mpath_persist.h"
74
75 #include "prioritizers/alua_rtpg.h"
76
77 #include "main.h"
78 #include "pidfile.h"
79 #include "uxlsnr.h"
80 #include "uxclnt.h"
81 #include "cli.h"
82 #include "cli_handlers.h"
83 #include "lock.h"
84 #include "waiter.h"
85 #include "io_err_stat.h"
86 #include "wwids.h"
87 #include "../third-party/valgrind/drd.h"
88
89 #define FILE_NAME_SIZE 256
90 #define CMDSIZE 160
91
92 #define LOG_MSG(a, b) \
93 do { \
94         if (pp->offline) \
95                 condlog(a, "%s: %s - path offline", pp->mpp->alias, pp->dev); \
96         else if (strlen(b)) \
97                 condlog(a, "%s: %s - %s", pp->mpp->alias, pp->dev, b); \
98 } while(0)
99
100 struct mpath_event_param
101 {
102         char * devname;
103         struct multipath *mpp;
104 };
105
106 int logsink;
107 int uxsock_timeout;
108 int verbosity;
109 int bindings_read_only;
110 int ignore_new_devs;
111 enum daemon_status running_state = DAEMON_INIT;
112 pid_t daemon_pid;
113 pthread_mutex_t config_lock = PTHREAD_MUTEX_INITIALIZER;
114 pthread_cond_t config_cond;
115
116 /*
117  * global copy of vecs for use in sig handlers
118  */
119 struct vectors * gvecs;
120
121 struct udev * udev;
122
123 struct config *multipath_conf;
124
125 /* Local variables */
126 static volatile sig_atomic_t exit_sig;
127 static volatile sig_atomic_t reconfig_sig;
128 static volatile sig_atomic_t log_reset_sig;
129
130 const char *
131 daemon_status(void)
132 {
133         switch (running_state) {
134         case DAEMON_INIT:
135                 return "init";
136         case DAEMON_START:
137                 return "startup";
138         case DAEMON_CONFIGURE:
139                 return "configure";
140         case DAEMON_IDLE:
141                 return "idle";
142         case DAEMON_RUNNING:
143                 return "running";
144         case DAEMON_SHUTDOWN:
145                 return "shutdown";
146         }
147         return NULL;
148 }
149
150 /*
151  * I love you too, systemd ...
152  */
153 const char *
154 sd_notify_status(void)
155 {
156         switch (running_state) {
157         case DAEMON_INIT:
158                 return "STATUS=init";
159         case DAEMON_START:
160                 return "STATUS=startup";
161         case DAEMON_CONFIGURE:
162                 return "STATUS=configure";
163         case DAEMON_IDLE:
164         case DAEMON_RUNNING:
165                 return "STATUS=up";
166         case DAEMON_SHUTDOWN:
167                 return "STATUS=shutdown";
168         }
169         return NULL;
170 }
171
172 #ifdef USE_SYSTEMD
173 static void do_sd_notify(enum daemon_status old_state)
174 {
175         /*
176          * Checkerloop switches back and forth between idle and running state.
177          * No need to tell systemd each time.
178          * These notifications cause a lot of overhead on dbus.
179          */
180         if ((running_state == DAEMON_IDLE || running_state == DAEMON_RUNNING) &&
181             (old_state == DAEMON_IDLE || old_state == DAEMON_RUNNING))
182                 return;
183         sd_notify(0, sd_notify_status());
184 }
185 #endif
186
187 static void config_cleanup(void *arg)
188 {
189         pthread_mutex_unlock(&config_lock);
190 }
191
192 void post_config_state(enum daemon_status state)
193 {
194         pthread_mutex_lock(&config_lock);
195         if (state != running_state) {
196                 enum daemon_status old_state = running_state;
197
198                 running_state = state;
199                 pthread_cond_broadcast(&config_cond);
200 #ifdef USE_SYSTEMD
201                 do_sd_notify(old_state);
202 #endif
203         }
204         pthread_mutex_unlock(&config_lock);
205 }
206
207 int set_config_state(enum daemon_status state)
208 {
209         int rc = 0;
210
211         pthread_cleanup_push(config_cleanup, NULL);
212         pthread_mutex_lock(&config_lock);
213         if (running_state != state) {
214                 enum daemon_status old_state = running_state;
215
216                 if (running_state != DAEMON_IDLE) {
217                         struct timespec ts;
218
219                         clock_gettime(CLOCK_MONOTONIC, &ts);
220                         ts.tv_sec += 1;
221                         rc = pthread_cond_timedwait(&config_cond,
222                                                     &config_lock, &ts);
223                 }
224                 if (!rc) {
225                         running_state = state;
226                         pthread_cond_broadcast(&config_cond);
227 #ifdef USE_SYSTEMD
228                         do_sd_notify(old_state);
229 #endif
230                 }
231         }
232         pthread_cleanup_pop(1);
233         return rc;
234 }
235
236 struct config *get_multipath_config(void)
237 {
238         rcu_read_lock();
239         return rcu_dereference(multipath_conf);
240 }
241
242 void put_multipath_config(struct config *conf)
243 {
244         rcu_read_unlock();
245 }
246
247 static int
248 need_switch_pathgroup (struct multipath * mpp, int refresh)
249 {
250         struct pathgroup * pgp;
251         struct path * pp;
252         unsigned int i, j;
253         struct config *conf;
254
255         if (!mpp || mpp->pgfailback == -FAILBACK_MANUAL)
256                 return 0;
257
258         /*
259          * Refresh path priority values
260          */
261         if (refresh) {
262                 vector_foreach_slot (mpp->pg, pgp, i) {
263                         vector_foreach_slot (pgp->paths, pp, j) {
264                                 conf = get_multipath_config();
265                                 pathinfo(pp, conf, DI_PRIO);
266                                 put_multipath_config(conf);
267                         }
268                 }
269         }
270
271         if (!mpp->pg || VECTOR_SIZE(mpp->paths) == 0)
272                 return 0;
273
274         mpp->bestpg = select_path_group(mpp);
275
276         if (mpp->bestpg != mpp->nextpg)
277                 return 1;
278
279         return 0;
280 }
281
282 static void
283 switch_pathgroup (struct multipath * mpp)
284 {
285         mpp->stat_switchgroup++;
286         dm_switchgroup(mpp->alias, mpp->bestpg);
287         condlog(2, "%s: switch to path group #%i",
288                  mpp->alias, mpp->bestpg);
289 }
290
291 static int
292 coalesce_maps(struct vectors *vecs, vector nmpv)
293 {
294         struct multipath * ompp;
295         vector ompv = vecs->mpvec;
296         unsigned int i, reassign_maps;
297         struct config *conf;
298
299         conf = get_multipath_config();
300         reassign_maps = conf->reassign_maps;
301         put_multipath_config(conf);
302         vector_foreach_slot (ompv, ompp, i) {
303                 condlog(3, "%s: coalesce map", ompp->alias);
304                 if (!find_mp_by_wwid(nmpv, ompp->wwid)) {
305                         /*
306                          * remove all current maps not allowed by the
307                          * current configuration
308                          */
309                         if (dm_flush_map(ompp->alias)) {
310                                 condlog(0, "%s: unable to flush devmap",
311                                         ompp->alias);
312                                 /*
313                                  * may be just because the device is open
314                                  */
315                                 if (setup_multipath(vecs, ompp) != 0) {
316                                         i--;
317                                         continue;
318                                 }
319                                 if (!vector_alloc_slot(nmpv))
320                                         return 1;
321
322                                 vector_set_slot(nmpv, ompp);
323
324                                 vector_del_slot(ompv, i);
325                                 i--;
326                         }
327                         else {
328                                 dm_lib_release();
329                                 condlog(2, "%s devmap removed", ompp->alias);
330                         }
331                 } else if (reassign_maps) {
332                         condlog(3, "%s: Reassign existing device-mapper"
333                                 " devices", ompp->alias);
334                         dm_reassign(ompp->alias);
335                 }
336         }
337         return 0;
338 }
339
340 static void
341 sync_maps_state(vector mpvec)
342 {
343         unsigned int i;
344         struct multipath *mpp;
345
346         vector_foreach_slot (mpvec, mpp, i)
347                 sync_map_state(mpp);
348 }
349
350 static int
351 flush_map(struct multipath * mpp, struct vectors * vecs, int nopaths)
352 {
353         int r;
354
355         if (nopaths)
356                 r = dm_flush_map_nopaths(mpp->alias, mpp->deferred_remove);
357         else
358                 r = dm_flush_map(mpp->alias);
359         /*
360          * clear references to this map before flushing so we can ignore
361          * the spurious uevent we may generate with the dm_flush_map call below
362          */
363         if (r) {
364                 /*
365                  * May not really be an error -- if the map was already flushed
366                  * from the device mapper by dmsetup(8) for instance.
367                  */
368                 if (r == 1)
369                         condlog(0, "%s: can't flush", mpp->alias);
370                 else {
371                         condlog(2, "%s: devmap deferred remove", mpp->alias);
372                         mpp->deferred_remove = DEFERRED_REMOVE_IN_PROGRESS;
373                 }
374                 return r;
375         }
376         else {
377                 dm_lib_release();
378                 condlog(2, "%s: map flushed", mpp->alias);
379         }
380
381         orphan_paths(vecs->pathvec, mpp);
382         remove_map_and_stop_waiter(mpp, vecs, 1);
383
384         return 0;
385 }
386
387 static int
388 uev_add_map (struct uevent * uev, struct vectors * vecs)
389 {
390         const char *alias;
391         int major = -1, minor = -1, rc;
392
393         condlog(3, "%s: add map (uevent)", uev->kernel);
394         alias = uevent_get_dm_name(uev);
395         if (!alias) {
396                 condlog(3, "%s: No DM_NAME in uevent", uev->kernel);
397                 major = uevent_get_major(uev);
398                 minor = uevent_get_minor(uev);
399                 alias = dm_mapname(major, minor);
400                 if (!alias) {
401                         condlog(2, "%s: mapname not found for %d:%d",
402                                 uev->kernel, major, minor);
403                         return 1;
404                 }
405         }
406         pthread_cleanup_push(cleanup_lock, &vecs->lock);
407         lock(&vecs->lock);
408         pthread_testcancel();
409         rc = ev_add_map(uev->kernel, alias, vecs);
410         lock_cleanup_pop(vecs->lock);
411         FREE(alias);
412         return rc;
413 }
414
415 /*
416  * ev_add_map expects that the multipath device already exists in kernel
417  * before it is called. It just adds a device to multipathd or updates an
418  * existing device.
419  */
420 int
421 ev_add_map (char * dev, const char * alias, struct vectors * vecs)
422 {
423         struct multipath * mpp;
424         int delayed_reconfig, reassign_maps;
425         struct config *conf;
426
427         if (!dm_is_mpath(alias)) {
428                 condlog(4, "%s: not a multipath map", alias);
429                 return 0;
430         }
431
432         mpp = find_mp_by_alias(vecs->mpvec, alias);
433
434         if (mpp) {
435                 if (mpp->wait_for_udev > 1) {
436                         condlog(2, "%s: performing delayed actions",
437                                 mpp->alias);
438                         if (update_map(mpp, vecs))
439                                 /* setup multipathd removed the map */
440                                 return 1;
441                 }
442                 conf = get_multipath_config();
443                 delayed_reconfig = conf->delayed_reconfig;
444                 reassign_maps = conf->reassign_maps;
445                 put_multipath_config(conf);
446                 if (mpp->wait_for_udev) {
447                         mpp->wait_for_udev = 0;
448                         if (delayed_reconfig &&
449                             !need_to_delay_reconfig(vecs)) {
450                                 condlog(2, "reconfigure (delayed)");
451                                 set_config_state(DAEMON_CONFIGURE);
452                                 return 0;
453                         }
454                 }
455                 /*
456                  * Not really an error -- we generate our own uevent
457                  * if we create a multipath mapped device as a result
458                  * of uev_add_path
459                  */
460                 if (reassign_maps) {
461                         condlog(3, "%s: Reassign existing device-mapper devices",
462                                 alias);
463                         dm_reassign(alias);
464                 }
465                 return 0;
466         }
467         condlog(2, "%s: adding map", alias);
468
469         /*
470          * now we can register the map
471          */
472         if ((mpp = add_map_without_path(vecs, alias))) {
473                 sync_map_state(mpp);
474                 condlog(2, "%s: devmap %s registered", alias, dev);
475                 return 0;
476         } else {
477                 condlog(2, "%s: ev_add_map failed", dev);
478                 return 1;
479         }
480 }
481
482 static int
483 uev_remove_map (struct uevent * uev, struct vectors * vecs)
484 {
485         const char *alias;
486         int minor;
487         struct multipath *mpp;
488
489         condlog(2, "%s: remove map (uevent)", uev->kernel);
490         alias = uevent_get_dm_name(uev);
491         if (!alias) {
492                 condlog(3, "%s: No DM_NAME in uevent, ignoring", uev->kernel);
493                 return 0;
494         }
495         minor = uevent_get_minor(uev);
496
497         pthread_cleanup_push(cleanup_lock, &vecs->lock);
498         lock(&vecs->lock);
499         pthread_testcancel();
500         mpp = find_mp_by_minor(vecs->mpvec, minor);
501
502         if (!mpp) {
503                 condlog(2, "%s: devmap not registered, can't remove",
504                         uev->kernel);
505                 goto out;
506         }
507         if (strcmp(mpp->alias, alias)) {
508                 condlog(2, "%s: minor number mismatch (map %d, event %d)",
509                         mpp->alias, mpp->dmi->minor, minor);
510                 goto out;
511         }
512
513         orphan_paths(vecs->pathvec, mpp);
514         remove_map_and_stop_waiter(mpp, vecs, 1);
515 out:
516         lock_cleanup_pop(vecs->lock);
517         FREE(alias);
518         return 0;
519 }
520
521 /* Called from CLI handler */
522 int
523 ev_remove_map (char * devname, char * alias, int minor, struct vectors * vecs)
524 {
525         struct multipath * mpp;
526
527         mpp = find_mp_by_minor(vecs->mpvec, minor);
528
529         if (!mpp) {
530                 condlog(2, "%s: devmap not registered, can't remove",
531                         devname);
532                 return 1;
533         }
534         if (strcmp(mpp->alias, alias)) {
535                 condlog(2, "%s: minor number mismatch (map %d, event %d)",
536                         mpp->alias, mpp->dmi->minor, minor);
537                 return 1;
538         }
539         return flush_map(mpp, vecs, 0);
540 }
541
542 static int
543 uev_add_path (struct uevent *uev, struct vectors * vecs, int need_do_map)
544 {
545         struct path *pp;
546         int ret = 0, i;
547         struct config *conf;
548
549         condlog(2, "%s: add path (uevent)", uev->kernel);
550         if (strstr(uev->kernel, "..") != NULL) {
551                 /*
552                  * Don't allow relative device names in the pathvec
553                  */
554                 condlog(0, "%s: path name is invalid", uev->kernel);
555                 return 1;
556         }
557
558         pthread_cleanup_push(cleanup_lock, &vecs->lock);
559         lock(&vecs->lock);
560         pthread_testcancel();
561         pp = find_path_by_dev(vecs->pathvec, uev->kernel);
562         if (pp) {
563                 int r;
564
565                 condlog(2, "%s: spurious uevent, path already in pathvec",
566                         uev->kernel);
567                 if (!pp->mpp && !strlen(pp->wwid)) {
568                         condlog(3, "%s: reinitialize path", uev->kernel);
569                         udev_device_unref(pp->udev);
570                         pp->udev = udev_device_ref(uev->udev);
571                         conf = get_multipath_config();
572                         r = pathinfo(pp, conf,
573                                      DI_ALL | DI_BLACKLIST);
574                         put_multipath_config(conf);
575                         if (r == PATHINFO_OK)
576                                 ret = ev_add_path(pp, vecs, need_do_map);
577                         else if (r == PATHINFO_SKIPPED) {
578                                 condlog(3, "%s: remove blacklisted path",
579                                         uev->kernel);
580                                 i = find_slot(vecs->pathvec, (void *)pp);
581                                 if (i != -1)
582                                         vector_del_slot(vecs->pathvec, i);
583                                 free_path(pp);
584                         } else {
585                                 condlog(0, "%s: failed to reinitialize path",
586                                         uev->kernel);
587                                 ret = 1;
588                         }
589                 }
590         }
591         lock_cleanup_pop(vecs->lock);
592         if (pp)
593                 return ret;
594
595         /*
596          * get path vital state
597          */
598         conf = get_multipath_config();
599         ret = alloc_path_with_pathinfo(conf, uev->udev,
600                                        uev->wwid, DI_ALL, &pp);
601         put_multipath_config(conf);
602         if (!pp) {
603                 if (ret == PATHINFO_SKIPPED)
604                         return 0;
605                 condlog(3, "%s: failed to get path info", uev->kernel);
606                 return 1;
607         }
608         pthread_cleanup_push(cleanup_lock, &vecs->lock);
609         lock(&vecs->lock);
610         pthread_testcancel();
611         ret = store_path(vecs->pathvec, pp);
612         if (!ret) {
613                 conf = get_multipath_config();
614                 pp->checkint = conf->checkint;
615                 put_multipath_config(conf);
616                 ret = ev_add_path(pp, vecs, need_do_map);
617         } else {
618                 condlog(0, "%s: failed to store path info, "
619                         "dropping event",
620                         uev->kernel);
621                 free_path(pp);
622                 ret = 1;
623         }
624         lock_cleanup_pop(vecs->lock);
625         return ret;
626 }
627
628 /*
629  * returns:
630  * 0: added
631  * 1: error
632  */
633 int
634 ev_add_path (struct path * pp, struct vectors * vecs, int need_do_map)
635 {
636         struct multipath * mpp;
637         char params[PARAMS_SIZE] = {0};
638         int retries = 3;
639         int start_waiter = 0;
640         int ret;
641
642         /*
643          * need path UID to go any further
644          */
645         if (strlen(pp->wwid) == 0) {
646                 condlog(0, "%s: failed to get path uid", pp->dev);
647                 goto fail; /* leave path added to pathvec */
648         }
649         mpp = find_mp_by_wwid(vecs->mpvec, pp->wwid);
650         if (mpp && mpp->wait_for_udev &&
651             (pathcount(mpp, PATH_UP) > 0 ||
652              (pathcount(mpp, PATH_GHOST) > 0 && pp->tpgs != TPGS_IMPLICIT &&
653               mpp->ghost_delay_tick <= 0))) {
654                 /* if wait_for_udev is set and valid paths exist */
655                 condlog(2, "%s: delaying path addition until %s is fully initialized", pp->dev, mpp->alias);
656                 mpp->wait_for_udev = 2;
657                 orphan_path(pp, "waiting for create to complete");
658                 return 0;
659         }
660
661         pp->mpp = mpp;
662 rescan:
663         if (mpp) {
664                 if (pp->size && mpp->size != pp->size) {
665                         condlog(0, "%s: failed to add new path %s, "
666                                 "device size mismatch",
667                                 mpp->alias, pp->dev);
668                         int i = find_slot(vecs->pathvec, (void *)pp);
669                         if (i != -1)
670                                 vector_del_slot(vecs->pathvec, i);
671                         free_path(pp);
672                         return 1;
673                 }
674
675                 condlog(4,"%s: adopting all paths for path %s",
676                         mpp->alias, pp->dev);
677                 if (adopt_paths(vecs->pathvec, mpp))
678                         goto fail; /* leave path added to pathvec */
679
680                 verify_paths(mpp, vecs);
681                 mpp->action = ACT_RELOAD;
682                 extract_hwe_from_path(mpp);
683         } else {
684                 if (!should_multipath(pp, vecs->pathvec)) {
685                         orphan_path(pp, "only one path");
686                         return 0;
687                 }
688                 condlog(4,"%s: creating new map", pp->dev);
689                 if ((mpp = add_map_with_path(vecs, pp, 1))) {
690                         mpp->action = ACT_CREATE;
691                         /*
692                          * We don't depend on ACT_CREATE, as domap will
693                          * set it to ACT_NOTHING when complete.
694                          */
695                         start_waiter = 1;
696                 }
697                 if (!start_waiter)
698                         goto fail; /* leave path added to pathvec */
699         }
700
701         /* persistent reservation check*/
702         mpath_pr_event_handle(pp);
703
704         if (!need_do_map)
705                 return 0;
706
707         if (!dm_map_present(mpp->alias)) {
708                 mpp->action = ACT_CREATE;
709                 start_waiter = 1;
710         }
711         /*
712          * push the map to the device-mapper
713          */
714         if (setup_map(mpp, params, PARAMS_SIZE)) {
715                 condlog(0, "%s: failed to setup map for addition of new "
716                         "path %s", mpp->alias, pp->dev);
717                 goto fail_map;
718         }
719         /*
720          * reload the map for the multipath mapped device
721          */
722 retry:
723         ret = domap(mpp, params, 1);
724         if (ret <= 0) {
725                 if (ret < 0 && retries-- > 0) {
726                         condlog(0, "%s: retry domap for addition of new "
727                                 "path %s", mpp->alias, pp->dev);
728                         sleep(1);
729                         goto retry;
730                 }
731                 condlog(0, "%s: failed in domap for addition of new "
732                         "path %s", mpp->alias, pp->dev);
733                 /*
734                  * deal with asynchronous uevents :((
735                  */
736                 if (mpp->action == ACT_RELOAD && retries-- > 0) {
737                         condlog(0, "%s: ev_add_path sleep", mpp->alias);
738                         sleep(1);
739                         update_mpp_paths(mpp, vecs->pathvec);
740                         goto rescan;
741                 }
742                 else if (mpp->action == ACT_RELOAD)
743                         condlog(0, "%s: giving up reload", mpp->alias);
744                 else
745                         goto fail_map;
746         }
747         dm_lib_release();
748
749         /*
750          * update our state from kernel regardless of create or reload
751          */
752         if (setup_multipath(vecs, mpp))
753                 goto fail; /* if setup_multipath fails, it removes the map */
754
755         sync_map_state(mpp);
756
757         if ((mpp->action == ACT_CREATE ||
758              (mpp->action == ACT_NOTHING && start_waiter && !mpp->waiter)) &&
759             start_waiter_thread(mpp, vecs))
760                         goto fail_map;
761
762         if (retries >= 0) {
763                 condlog(2, "%s [%s]: path added to devmap %s",
764                         pp->dev, pp->dev_t, mpp->alias);
765                 return 0;
766         } else
767                 goto fail;
768
769 fail_map:
770         remove_map(mpp, vecs, 1);
771 fail:
772         orphan_path(pp, "failed to add path");
773         return 1;
774 }
775
776 static int
777 uev_remove_path (struct uevent *uev, struct vectors * vecs, int need_do_map)
778 {
779         struct path *pp;
780         int ret;
781
782         condlog(2, "%s: remove path (uevent)", uev->kernel);
783         pthread_cleanup_push(cleanup_lock, &vecs->lock);
784         lock(&vecs->lock);
785         pthread_testcancel();
786         pp = find_path_by_dev(vecs->pathvec, uev->kernel);
787         if (pp)
788                 ret = ev_remove_path(pp, vecs, need_do_map);
789         lock_cleanup_pop(vecs->lock);
790         if (!pp) {
791                 /* Not an error; path might have been purged earlier */
792                 condlog(0, "%s: path already removed", uev->kernel);
793                 return 0;
794         }
795         return ret;
796 }
797
798 int
799 ev_remove_path (struct path *pp, struct vectors * vecs, int need_do_map)
800 {
801         struct multipath * mpp;
802         int i, retval = 0;
803         char params[PARAMS_SIZE] = {0};
804
805         /*
806          * avoid referring to the map of an orphaned path
807          */
808         if ((mpp = pp->mpp)) {
809                 /*
810                  * transform the mp->pg vector of vectors of paths
811                  * into a mp->params string to feed the device-mapper
812                  */
813                 if (update_mpp_paths(mpp, vecs->pathvec)) {
814                         condlog(0, "%s: failed to update paths",
815                                 mpp->alias);
816                         goto fail;
817                 }
818                 if ((i = find_slot(mpp->paths, (void *)pp)) != -1)
819                         vector_del_slot(mpp->paths, i);
820
821                 /*
822                  * remove the map IFF removing the last path
823                  */
824                 if (VECTOR_SIZE(mpp->paths) == 0) {
825                         char alias[WWID_SIZE];
826
827                         /*
828                          * flush_map will fail if the device is open
829                          */
830                         strncpy(alias, mpp->alias, WWID_SIZE);
831                         if (mpp->flush_on_last_del == FLUSH_ENABLED) {
832                                 condlog(2, "%s Last path deleted, disabling queueing", mpp->alias);
833                                 mpp->retry_tick = 0;
834                                 mpp->no_path_retry = NO_PATH_RETRY_FAIL;
835                                 mpp->disable_queueing = 1;
836                                 mpp->stat_map_failures++;
837                                 dm_queue_if_no_path(mpp->alias, 0);
838                         }
839                         if (!flush_map(mpp, vecs, 1)) {
840                                 condlog(2, "%s: removed map after"
841                                         " removing all paths",
842                                         alias);
843                                 retval = 0;
844                                 goto out;
845                         }
846                         /*
847                          * Not an error, continue
848                          */
849                 }
850
851                 if (setup_map(mpp, params, PARAMS_SIZE)) {
852                         condlog(0, "%s: failed to setup map for"
853                                 " removal of path %s", mpp->alias, pp->dev);
854                         goto fail;
855                 }
856
857                 if (mpp->wait_for_udev) {
858                         mpp->wait_for_udev = 2;
859                         goto out;
860                 }
861
862                 if (!need_do_map)
863                         goto out;
864                 /*
865                  * reload the map
866                  */
867                 mpp->action = ACT_RELOAD;
868                 if (domap(mpp, params, 1) <= 0) {
869                         condlog(0, "%s: failed in domap for "
870                                 "removal of path %s",
871                                 mpp->alias, pp->dev);
872                         retval = 1;
873                 } else {
874                         /*
875                          * update our state from kernel
876                          */
877                         if (setup_multipath(vecs, mpp))
878                                 return 1;
879                         sync_map_state(mpp);
880
881                         condlog(2, "%s [%s]: path removed from map %s",
882                                 pp->dev, pp->dev_t, mpp->alias);
883                 }
884         }
885
886 out:
887         if ((i = find_slot(vecs->pathvec, (void *)pp)) != -1)
888                 vector_del_slot(vecs->pathvec, i);
889
890         free_path(pp);
891
892         return retval;
893
894 fail:
895         remove_map_and_stop_waiter(mpp, vecs, 1);
896         return 1;
897 }
898
899 static int
900 uev_update_path (struct uevent *uev, struct vectors * vecs)
901 {
902         int ro, retval = 0;
903         struct path * pp;
904         struct config *conf;
905         int disable_changed_wwids;
906         int needs_reinit = 0;
907
908         conf = get_multipath_config();
909         disable_changed_wwids = conf->disable_changed_wwids;
910         put_multipath_config(conf);
911
912         ro = uevent_get_disk_ro(uev);
913
914         pthread_cleanup_push(cleanup_lock, &vecs->lock);
915         lock(&vecs->lock);
916         pthread_testcancel();
917
918         pp = find_path_by_dev(vecs->pathvec, uev->kernel);
919         if (pp) {
920                 struct multipath *mpp = pp->mpp;
921
922                 if (disable_changed_wwids &&
923                     (strlen(pp->wwid) || pp->wwid_changed)) {
924                         char wwid[WWID_SIZE];
925
926                         strcpy(wwid, pp->wwid);
927                         get_uid(pp, pp->state, uev->udev);
928                         if (strcmp(wwid, pp->wwid) != 0) {
929                                 condlog(0, "%s: path wwid changed from '%s' to '%s'. disallowing", uev->kernel, wwid, pp->wwid);
930                                 strcpy(pp->wwid, wwid);
931                                 if (!pp->wwid_changed) {
932                                         pp->wwid_changed = 1;
933                                         pp->tick = 1;
934                                         if (pp->mpp)
935                                                 dm_fail_path(pp->mpp->alias, pp->dev_t);
936                                 }
937                                 goto out;
938                         } else
939                                 pp->wwid_changed = 0;
940                 }
941
942                 if (pp->initialized == INIT_REQUESTED_UDEV)
943                         needs_reinit = 1;
944                 else if (mpp && ro >= 0) {
945                         condlog(2, "%s: update path write_protect to '%d' (uevent)", uev->kernel, ro);
946
947                         if (mpp->wait_for_udev)
948                                 mpp->wait_for_udev = 2;
949                         else {
950                                 if (ro == 1)
951                                         pp->mpp->force_readonly = 1;
952                                 retval = reload_map(vecs, mpp, 0, 1);
953                                 pp->mpp->force_readonly = 0;
954                                 condlog(2, "%s: map %s reloaded (retval %d)",
955                                         uev->kernel, mpp->alias, retval);
956                         }
957                 }
958         }
959 out:
960         lock_cleanup_pop(vecs->lock);
961         if (!pp) {
962                 /* If the path is blacklisted, print a debug/non-default verbosity message. */
963                 if (uev->udev) {
964                         int flag = DI_SYSFS | DI_WWID;
965
966                         conf = get_multipath_config();
967                         retval = alloc_path_with_pathinfo(conf, uev->udev, uev->wwid, flag, NULL);
968                         put_multipath_config(conf);
969
970                         if (retval == PATHINFO_SKIPPED) {
971                                 condlog(3, "%s: spurious uevent, path is blacklisted", uev->kernel);
972                                 return 0;
973                         }
974                 }
975
976                 condlog(0, "%s: spurious uevent, path not found", uev->kernel);
977         }
978         if (needs_reinit)
979                 retval = uev_add_path(uev, vecs, 1);
980         return retval;
981 }
982
983 static int
984 uev_pathfail_check(struct uevent *uev, struct vectors *vecs)
985 {
986         const char *action = NULL, *devt = NULL;
987         struct path *pp;
988         int r = 1;
989
990         action = uevent_get_dm_action(uev);
991         if (!action)
992                 return 1;
993         if (strncmp(action, "PATH_FAILED", 11))
994                 goto out;
995         devt = uevent_get_dm_path(uev);
996         if (!devt) {
997                 condlog(3, "%s: No DM_PATH in uevent", uev->kernel);
998                 goto out;
999         }
1000
1001         pthread_cleanup_push(cleanup_lock, &vecs->lock);
1002         lock(&vecs->lock);
1003         pthread_testcancel();
1004         pp = find_path_by_devt(vecs->pathvec, devt);
1005         if (!pp)
1006                 goto out_lock;
1007         r = io_err_stat_handle_pathfail(pp);
1008         if (r)
1009                 condlog(3, "io_err_stat: %s: cannot handle pathfail uevent",
1010                                 pp->dev);
1011 out_lock:
1012         lock_cleanup_pop(vecs->lock);
1013         FREE(devt);
1014         FREE(action);
1015         return r;
1016 out:
1017         FREE(action);
1018         return 1;
1019 }
1020
1021 static int
1022 map_discovery (struct vectors * vecs)
1023 {
1024         struct multipath * mpp;
1025         unsigned int i;
1026
1027         if (dm_get_maps(vecs->mpvec))
1028                 return 1;
1029
1030         vector_foreach_slot (vecs->mpvec, mpp, i)
1031                 if (update_multipath_table(mpp, vecs->pathvec, 1) ||
1032                     update_multipath_status(mpp)) {
1033                         remove_map(mpp, vecs, 1);
1034                         i--;
1035                 }
1036
1037         return 0;
1038 }
1039
1040 int
1041 uxsock_trigger (char * str, char ** reply, int * len, bool is_root,
1042                 void * trigger_data)
1043 {
1044         struct vectors * vecs;
1045         int r;
1046
1047         *reply = NULL;
1048         *len = 0;
1049         vecs = (struct vectors *)trigger_data;
1050
1051         if ((str != NULL) && (is_root == false) &&
1052             (strncmp(str, "list", strlen("list")) != 0) &&
1053             (strncmp(str, "show", strlen("show")) != 0)) {
1054                 *reply = STRDUP("permission deny: need to be root");
1055                 if (*reply)
1056                         *len = strlen(*reply) + 1;
1057                 return 1;
1058         }
1059
1060         r = parse_cmd(str, reply, len, vecs, uxsock_timeout / 1000);
1061
1062         if (r > 0) {
1063                 if (r == ETIMEDOUT)
1064                         *reply = STRDUP("timeout\n");
1065                 else
1066                         *reply = STRDUP("fail\n");
1067                 if (*reply)
1068                         *len = strlen(*reply) + 1;
1069                 r = 1;
1070         }
1071         else if (!r && *len == 0) {
1072                 *reply = STRDUP("ok\n");
1073                 if (*reply)
1074                         *len = strlen(*reply) + 1;
1075                 r = 0;
1076         }
1077         /* else if (r < 0) leave *reply alone */
1078
1079         return r;
1080 }
1081
1082 int
1083 uev_trigger (struct uevent * uev, void * trigger_data)
1084 {
1085         int r = 0;
1086         struct vectors * vecs;
1087         struct uevent *merge_uev, *tmp;
1088
1089         vecs = (struct vectors *)trigger_data;
1090
1091         pthread_cleanup_push(config_cleanup, NULL);
1092         pthread_mutex_lock(&config_lock);
1093         if (running_state != DAEMON_IDLE &&
1094             running_state != DAEMON_RUNNING)
1095                 pthread_cond_wait(&config_cond, &config_lock);
1096         pthread_cleanup_pop(1);
1097
1098         if (running_state == DAEMON_SHUTDOWN)
1099                 return 0;
1100
1101         /*
1102          * device map event
1103          * Add events are ignored here as the tables
1104          * are not fully initialised then.
1105          */
1106         if (!strncmp(uev->kernel, "dm-", 3)) {
1107                 if (!uevent_is_mpath(uev))
1108                         goto out;
1109                 if (!strncmp(uev->action, "change", 6)) {
1110                         r = uev_add_map(uev, vecs);
1111
1112                         /*
1113                          * the kernel-side dm-mpath issues a PATH_FAILED event
1114                          * when it encounters a path IO error. It is reason-
1115                          * able be the entry of path IO error accounting pro-
1116                          * cess.
1117                          */
1118                         uev_pathfail_check(uev, vecs);
1119                 } else if (!strncmp(uev->action, "remove", 6)) {
1120                         r = uev_remove_map(uev, vecs);
1121                 }
1122                 goto out;
1123         }
1124
1125         /*
1126          * path add/remove/change event, add/remove maybe merged
1127          */
1128         list_for_each_entry_safe(merge_uev, tmp, &uev->merge_node, node) {
1129                 if (!strncmp(merge_uev->action, "add", 3))
1130                         r += uev_add_path(merge_uev, vecs, 0);
1131                 if (!strncmp(merge_uev->action, "remove", 6))
1132                         r += uev_remove_path(merge_uev, vecs, 0);
1133         }
1134
1135         if (!strncmp(uev->action, "add", 3))
1136                 r += uev_add_path(uev, vecs, 1);
1137         if (!strncmp(uev->action, "remove", 6))
1138                 r += uev_remove_path(uev, vecs, 1);
1139         if (!strncmp(uev->action, "change", 6))
1140                 r += uev_update_path(uev, vecs);
1141
1142 out:
1143         return r;
1144 }
1145
1146 static void rcu_unregister(void *param)
1147 {
1148         rcu_unregister_thread();
1149 }
1150
1151 static void *
1152 ueventloop (void * ap)
1153 {
1154         struct udev *udev = ap;
1155
1156         pthread_cleanup_push(rcu_unregister, NULL);
1157         rcu_register_thread();
1158         if (uevent_listen(udev))
1159                 condlog(0, "error starting uevent listener");
1160         pthread_cleanup_pop(1);
1161         return NULL;
1162 }
1163
1164 static void *
1165 uevqloop (void * ap)
1166 {
1167         pthread_cleanup_push(rcu_unregister, NULL);
1168         rcu_register_thread();
1169         if (uevent_dispatch(&uev_trigger, ap))
1170                 condlog(0, "error starting uevent dispatcher");
1171         pthread_cleanup_pop(1);
1172         return NULL;
1173 }
1174 static void *
1175 uxlsnrloop (void * ap)
1176 {
1177         if (cli_init()) {
1178                 condlog(1, "Failed to init uxsock listener");
1179                 return NULL;
1180         }
1181         pthread_cleanup_push(rcu_unregister, NULL);
1182         rcu_register_thread();
1183         set_handler_callback(LIST+PATHS, cli_list_paths);
1184         set_handler_callback(LIST+PATHS+FMT, cli_list_paths_fmt);
1185         set_handler_callback(LIST+PATHS+RAW+FMT, cli_list_paths_raw);
1186         set_handler_callback(LIST+PATH, cli_list_path);
1187         set_handler_callback(LIST+MAPS, cli_list_maps);
1188         set_unlocked_handler_callback(LIST+STATUS, cli_list_status);
1189         set_unlocked_handler_callback(LIST+DAEMON, cli_list_daemon);
1190         set_handler_callback(LIST+MAPS+STATUS, cli_list_maps_status);
1191         set_handler_callback(LIST+MAPS+STATS, cli_list_maps_stats);
1192         set_handler_callback(LIST+MAPS+FMT, cli_list_maps_fmt);
1193         set_handler_callback(LIST+MAPS+RAW+FMT, cli_list_maps_raw);
1194         set_handler_callback(LIST+MAPS+TOPOLOGY, cli_list_maps_topology);
1195         set_handler_callback(LIST+TOPOLOGY, cli_list_maps_topology);
1196         set_handler_callback(LIST+MAPS+JSON, cli_list_maps_json);
1197         set_handler_callback(LIST+MAP+TOPOLOGY, cli_list_map_topology);
1198         set_handler_callback(LIST+MAP+FMT, cli_list_map_fmt);
1199         set_handler_callback(LIST+MAP+RAW+FMT, cli_list_map_fmt);
1200         set_handler_callback(LIST+MAP+JSON, cli_list_map_json);
1201         set_handler_callback(LIST+CONFIG, cli_list_config);
1202         set_handler_callback(LIST+BLACKLIST, cli_list_blacklist);
1203         set_handler_callback(LIST+DEVICES, cli_list_devices);
1204         set_handler_callback(LIST+WILDCARDS, cli_list_wildcards);
1205         set_handler_callback(RESET+MAPS+STATS, cli_reset_maps_stats);
1206         set_handler_callback(RESET+MAP+STATS, cli_reset_map_stats);
1207         set_handler_callback(ADD+PATH, cli_add_path);
1208         set_handler_callback(DEL+PATH, cli_del_path);
1209         set_handler_callback(ADD+MAP, cli_add_map);
1210         set_handler_callback(DEL+MAP, cli_del_map);
1211         set_handler_callback(SWITCH+MAP+GROUP, cli_switch_group);
1212         set_unlocked_handler_callback(RECONFIGURE, cli_reconfigure);
1213         set_handler_callback(SUSPEND+MAP, cli_suspend);
1214         set_handler_callback(RESUME+MAP, cli_resume);
1215         set_handler_callback(RESIZE+MAP, cli_resize);
1216         set_handler_callback(RELOAD+MAP, cli_reload);
1217         set_handler_callback(RESET+MAP, cli_reassign);
1218         set_handler_callback(REINSTATE+PATH, cli_reinstate);
1219         set_handler_callback(FAIL+PATH, cli_fail);
1220         set_handler_callback(DISABLEQ+MAP, cli_disable_queueing);
1221         set_handler_callback(RESTOREQ+MAP, cli_restore_queueing);
1222         set_handler_callback(DISABLEQ+MAPS, cli_disable_all_queueing);
1223         set_handler_callback(RESTOREQ+MAPS, cli_restore_all_queueing);
1224         set_unlocked_handler_callback(QUIT, cli_quit);
1225         set_unlocked_handler_callback(SHUTDOWN, cli_shutdown);
1226         set_handler_callback(GETPRSTATUS+MAP, cli_getprstatus);
1227         set_handler_callback(SETPRSTATUS+MAP, cli_setprstatus);
1228         set_handler_callback(UNSETPRSTATUS+MAP, cli_unsetprstatus);
1229         set_handler_callback(FORCEQ+DAEMON, cli_force_no_daemon_q);
1230         set_handler_callback(RESTOREQ+DAEMON, cli_restore_no_daemon_q);
1231         set_handler_callback(GETPRKEY+MAP, cli_getprkey);
1232         set_handler_callback(SETPRKEY+MAP+KEY, cli_setprkey);
1233         set_handler_callback(UNSETPRKEY+MAP, cli_unsetprkey);
1234
1235         umask(077);
1236         uxsock_listen(&uxsock_trigger, ap);
1237         pthread_cleanup_pop(1);
1238         return NULL;
1239 }
1240
1241 void
1242 exit_daemon (void)
1243 {
1244         post_config_state(DAEMON_SHUTDOWN);
1245 }
1246
1247 static void
1248 fail_path (struct path * pp, int del_active)
1249 {
1250         if (!pp->mpp)
1251                 return;
1252
1253         condlog(2, "checker failed path %s in map %s",
1254                  pp->dev_t, pp->mpp->alias);
1255
1256         dm_fail_path(pp->mpp->alias, pp->dev_t);
1257         if (del_active)
1258                 update_queue_mode_del_path(pp->mpp);
1259 }
1260
1261 /*
1262  * caller must have locked the path list before calling that function
1263  */
1264 static int
1265 reinstate_path (struct path * pp, int add_active)
1266 {
1267         int ret = 0;
1268
1269         if (!pp->mpp)
1270                 return 0;
1271
1272         if (dm_reinstate_path(pp->mpp->alias, pp->dev_t)) {
1273                 condlog(0, "%s: reinstate failed", pp->dev_t);
1274                 ret = 1;
1275         } else {
1276                 condlog(2, "%s: reinstated", pp->dev_t);
1277                 if (add_active)
1278                         update_queue_mode_add_path(pp->mpp);
1279         }
1280         return ret;
1281 }
1282
1283 static void
1284 enable_group(struct path * pp)
1285 {
1286         struct pathgroup * pgp;
1287
1288         /*
1289          * if path is added through uev_add_path, pgindex can be unset.
1290          * next update_strings() will set it, upon map reload event.
1291          *
1292          * we can safely return here, because upon map reload, all
1293          * PG will be enabled.
1294          */
1295         if (!pp->mpp->pg || !pp->pgindex)
1296                 return;
1297
1298         pgp = VECTOR_SLOT(pp->mpp->pg, pp->pgindex - 1);
1299
1300         if (pgp->status == PGSTATE_DISABLED) {
1301                 condlog(2, "%s: enable group #%i", pp->mpp->alias, pp->pgindex);
1302                 dm_enablegroup(pp->mpp->alias, pp->pgindex);
1303         }
1304 }
1305
1306 static void
1307 mpvec_garbage_collector (struct vectors * vecs)
1308 {
1309         struct multipath * mpp;
1310         unsigned int i;
1311
1312         if (!vecs->mpvec)
1313                 return;
1314
1315         vector_foreach_slot (vecs->mpvec, mpp, i) {
1316                 if (mpp && mpp->alias && !dm_map_present(mpp->alias)) {
1317                         condlog(2, "%s: remove dead map", mpp->alias);
1318                         remove_map_and_stop_waiter(mpp, vecs, 1);
1319                         i--;
1320                 }
1321         }
1322 }
1323
1324 /* This is called after a path has started working again. It the multipath
1325  * device for this path uses the followover failback type, and this is the
1326  * best pathgroup, and this is the first path in the pathgroup to come back
1327  * up, then switch to this pathgroup */
1328 static int
1329 followover_should_failback(struct path * pp)
1330 {
1331         struct pathgroup * pgp;
1332         struct path *pp1;
1333         int i;
1334
1335         if (pp->mpp->pgfailback != -FAILBACK_FOLLOWOVER ||
1336             !pp->mpp->pg || !pp->pgindex ||
1337             pp->pgindex != pp->mpp->bestpg)
1338                 return 0;
1339
1340         pgp = VECTOR_SLOT(pp->mpp->pg, pp->pgindex - 1);
1341         vector_foreach_slot(pgp->paths, pp1, i) {
1342                 if (pp1 == pp)
1343                         continue;
1344                 if (pp1->chkrstate != PATH_DOWN && pp1->chkrstate != PATH_SHAKY)
1345                         return 0;
1346         }
1347         return 1;
1348 }
1349
1350 static void
1351 missing_uev_wait_tick(struct vectors *vecs)
1352 {
1353         struct multipath * mpp;
1354         unsigned int i;
1355         int timed_out = 0, delayed_reconfig;
1356         struct config *conf;
1357
1358         vector_foreach_slot (vecs->mpvec, mpp, i) {
1359                 if (mpp->wait_for_udev && --mpp->uev_wait_tick <= 0) {
1360                         timed_out = 1;
1361                         condlog(0, "%s: timeout waiting on creation uevent. enabling reloads", mpp->alias);
1362                         if (mpp->wait_for_udev > 1 && update_map(mpp, vecs)) {
1363                                 /* update_map removed map */
1364                                 i--;
1365                                 continue;
1366                         }
1367                         mpp->wait_for_udev = 0;
1368                 }
1369         }
1370
1371         conf = get_multipath_config();
1372         delayed_reconfig = conf->delayed_reconfig;
1373         put_multipath_config(conf);
1374         if (timed_out && delayed_reconfig &&
1375             !need_to_delay_reconfig(vecs)) {
1376                 condlog(2, "reconfigure (delayed)");
1377                 set_config_state(DAEMON_CONFIGURE);
1378         }
1379 }
1380
1381 static void
1382 ghost_delay_tick(struct vectors *vecs)
1383 {
1384         struct multipath * mpp;
1385         unsigned int i;
1386
1387         vector_foreach_slot (vecs->mpvec, mpp, i) {
1388                 if (mpp->ghost_delay_tick <= 0)
1389                         continue;
1390                 if (--mpp->ghost_delay_tick <= 0) {
1391                         condlog(0, "%s: timed out waiting for active path",
1392                                 mpp->alias);
1393                         mpp->force_udev_reload = 1;
1394                         if (update_map(mpp, vecs) != 0) {
1395                                 /* update_map removed map */
1396                                 i--;
1397                                 continue;
1398                         }
1399                 }
1400         }
1401 }
1402
1403 static void
1404 defered_failback_tick (vector mpvec)
1405 {
1406         struct multipath * mpp;
1407         unsigned int i;
1408
1409         vector_foreach_slot (mpvec, mpp, i) {
1410                 /*
1411                  * defered failback getting sooner
1412                  */
1413                 if (mpp->pgfailback > 0 && mpp->failback_tick > 0) {
1414                         mpp->failback_tick--;
1415
1416                         if (!mpp->failback_tick && need_switch_pathgroup(mpp, 1))
1417                                 switch_pathgroup(mpp);
1418                 }
1419         }
1420 }
1421
1422 static void
1423 retry_count_tick(vector mpvec)
1424 {
1425         struct multipath *mpp;
1426         unsigned int i;
1427
1428         vector_foreach_slot (mpvec, mpp, i) {
1429                 if (mpp->retry_tick > 0) {
1430                         mpp->stat_total_queueing_time++;
1431                         condlog(4, "%s: Retrying.. No active path", mpp->alias);
1432                         if(--mpp->retry_tick == 0) {
1433                                 mpp->stat_map_failures++;
1434                                 dm_queue_if_no_path(mpp->alias, 0);
1435                                 condlog(2, "%s: Disable queueing", mpp->alias);
1436                         }
1437                 }
1438         }
1439 }
1440
1441 int update_prio(struct path *pp, int refresh_all)
1442 {
1443         int oldpriority;
1444         struct path *pp1;
1445         struct pathgroup * pgp;
1446         int i, j, changed = 0;
1447         struct config *conf;
1448
1449         if (refresh_all) {
1450                 vector_foreach_slot (pp->mpp->pg, pgp, i) {
1451                         vector_foreach_slot (pgp->paths, pp1, j) {
1452                                 oldpriority = pp1->priority;
1453                                 conf = get_multipath_config();
1454                                 pathinfo(pp1, conf, DI_PRIO);
1455                                 put_multipath_config(conf);
1456                                 if (pp1->priority != oldpriority)
1457                                         changed = 1;
1458                         }
1459                 }
1460                 return changed;
1461         }
1462         oldpriority = pp->priority;
1463         conf = get_multipath_config();
1464         if (pp->state != PATH_DOWN)
1465                 pathinfo(pp, conf, DI_PRIO);
1466         put_multipath_config(conf);
1467
1468         if (pp->priority == oldpriority)
1469                 return 0;
1470         return 1;
1471 }
1472
1473 int update_path_groups(struct multipath *mpp, struct vectors *vecs, int refresh)
1474 {
1475         if (reload_map(vecs, mpp, refresh, 1))
1476                 return 1;
1477
1478         dm_lib_release();
1479         if (setup_multipath(vecs, mpp) != 0)
1480                 return 1;
1481         sync_map_state(mpp);
1482
1483         return 0;
1484 }
1485
1486 void repair_path(struct path * pp)
1487 {
1488         if (pp->state != PATH_DOWN)
1489                 return;
1490
1491         checker_repair(&pp->checker);
1492         LOG_MSG(1, checker_message(&pp->checker));
1493 }
1494
1495 /*
1496  * Returns '1' if the path has been checked, '-1' if it was blacklisted
1497  * and '0' otherwise
1498  */
1499 int
1500 check_path (struct vectors * vecs, struct path * pp, int ticks)
1501 {
1502         int newstate;
1503         int new_path_up = 0;
1504         int chkr_new_path_up = 0;
1505         int add_active;
1506         int disable_reinstate = 0;
1507         int oldchkrstate = pp->chkrstate;
1508         int retrigger_tries, checkint;
1509         struct config *conf;
1510         int ret;
1511
1512         if ((pp->initialized == INIT_OK ||
1513              pp->initialized == INIT_REQUESTED_UDEV) && !pp->mpp)
1514                 return 0;
1515
1516         if (pp->tick)
1517                 pp->tick -= (pp->tick > ticks) ? ticks : pp->tick;
1518         if (pp->tick)
1519                 return 0; /* don't check this path yet */
1520
1521         conf = get_multipath_config();
1522         retrigger_tries = conf->retrigger_tries;
1523         checkint = conf->checkint;
1524         put_multipath_config(conf);
1525         if (!pp->mpp && pp->initialized == INIT_MISSING_UDEV &&
1526             pp->retriggers < retrigger_tries) {
1527                 condlog(2, "%s: triggering change event to reinitialize",
1528                         pp->dev);
1529                 pp->initialized = INIT_REQUESTED_UDEV;
1530                 pp->retriggers++;
1531                 sysfs_attr_set_value(pp->udev, "uevent", "change",
1532                                      strlen("change"));
1533                 return 0;
1534         }
1535
1536         /*
1537          * provision a next check soonest,
1538          * in case we exit abnormaly from here
1539          */
1540         pp->tick = checkint;
1541
1542         newstate = path_offline(pp);
1543         /*
1544          * Wait for uevent for removed paths;
1545          * some LLDDs like zfcp keep paths unavailable
1546          * without sending uevents.
1547          */
1548         if (newstate == PATH_REMOVED)
1549                 newstate = PATH_DOWN;
1550
1551         if (newstate == PATH_UP) {
1552                 conf = get_multipath_config();
1553                 newstate = get_state(pp, conf, 1, newstate);
1554                 put_multipath_config(conf);
1555         } else
1556                 checker_clear_message(&pp->checker);
1557
1558         if (pp->wwid_changed) {
1559                 condlog(2, "%s: path wwid has changed. Refusing to use",
1560                         pp->dev);
1561                 newstate = PATH_DOWN;
1562         }
1563
1564         if (newstate == PATH_WILD || newstate == PATH_UNCHECKED) {
1565                 condlog(2, "%s: unusable path", pp->dev);
1566                 conf = get_multipath_config();
1567                 pathinfo(pp, conf, 0);
1568                 put_multipath_config(conf);
1569                 return 1;
1570         }
1571         if (!pp->mpp) {
1572                 if (!strlen(pp->wwid) && pp->initialized != INIT_MISSING_UDEV &&
1573                     (newstate == PATH_UP || newstate == PATH_GHOST)) {
1574                         condlog(2, "%s: add missing path", pp->dev);
1575                         conf = get_multipath_config();
1576                         ret = pathinfo(pp, conf, DI_ALL | DI_BLACKLIST);
1577                         if (ret == PATHINFO_OK) {
1578                                 ev_add_path(pp, vecs, 1);
1579                                 pp->tick = 1;
1580                         } else if (ret == PATHINFO_SKIPPED) {
1581                                 put_multipath_config(conf);
1582                                 return -1;
1583                         }
1584                         put_multipath_config(conf);
1585                 }
1586                 return 0;
1587         }
1588         /*
1589          * Async IO in flight. Keep the previous path state
1590          * and reschedule as soon as possible
1591          */
1592         if (newstate == PATH_PENDING) {
1593                 pp->tick = 1;
1594                 return 0;
1595         }
1596         /*
1597          * Synchronize with kernel state
1598          */
1599         if (update_multipath_strings(pp->mpp, vecs->pathvec, 1)) {
1600                 condlog(1, "%s: Could not synchronize with kernel state",
1601                         pp->dev);
1602                 pp->dmstate = PSTATE_UNDEF;
1603         }
1604         /* if update_multipath_strings orphaned the path, quit early */
1605         if (!pp->mpp)
1606                 return 0;
1607
1608         if (pp->io_err_disable_reinstate && hit_io_err_recheck_time(pp)) {
1609                 pp->state = PATH_SHAKY;
1610                 /*
1611                  * to reschedule as soon as possible,so that this path can
1612                  * be recoverd in time
1613                  */
1614                 pp->tick = 1;
1615                 return 1;
1616         }
1617
1618         if ((newstate == PATH_UP || newstate == PATH_GHOST) &&
1619              pp->wait_checks > 0) {
1620                 if (pp->mpp->nr_active > 0) {
1621                         pp->state = PATH_DELAYED;
1622                         pp->wait_checks--;
1623                         return 1;
1624                 } else
1625                         pp->wait_checks = 0;
1626         }
1627
1628         /*
1629          * don't reinstate failed path, if its in stand-by
1630          * and if target supports only implicit tpgs mode.
1631          * this will prevent unnecessary i/o by dm on stand-by
1632          * paths if there are no other active paths in map.
1633          */
1634         disable_reinstate = (newstate == PATH_GHOST &&
1635                             pp->mpp->nr_active == 0 &&
1636                             pp->tpgs == TPGS_IMPLICIT) ? 1 : 0;
1637
1638         pp->chkrstate = newstate;
1639         if (newstate != pp->state) {
1640                 int oldstate = pp->state;
1641                 pp->state = newstate;
1642
1643                 LOG_MSG(1, checker_message(&pp->checker));
1644
1645                 /*
1646                  * upon state change, reset the checkint
1647                  * to the shortest delay
1648                  */
1649                 conf = get_multipath_config();
1650                 pp->checkint = conf->checkint;
1651                 put_multipath_config(conf);
1652
1653                 if (newstate != PATH_UP && newstate != PATH_GHOST) {
1654                         /*
1655                          * proactively fail path in the DM
1656                          */
1657                         if (oldstate == PATH_UP ||
1658                             oldstate == PATH_GHOST) {
1659                                 fail_path(pp, 1);
1660                                 if (pp->mpp->delay_wait_checks > 0 &&
1661                                     pp->watch_checks > 0) {
1662                                         pp->wait_checks = pp->mpp->delay_wait_checks;
1663                                         pp->watch_checks = 0;
1664                                 }
1665                         }else
1666                                 fail_path(pp, 0);
1667
1668                         /*
1669                          * cancel scheduled failback
1670                          */
1671                         pp->mpp->failback_tick = 0;
1672
1673                         pp->mpp->stat_path_failures++;
1674                         repair_path(pp);
1675                         return 1;
1676                 }
1677
1678                 if(newstate == PATH_UP || newstate == PATH_GHOST){
1679                         if ( pp->mpp && pp->mpp->prflag ){
1680                                 /*
1681                                  * Check Persistent Reservation.
1682                                  */
1683                         condlog(2, "%s: checking persistent reservation "
1684                                 "registration", pp->dev);
1685                         mpath_pr_event_handle(pp);
1686                         }
1687                 }
1688
1689                 /*
1690                  * reinstate this path
1691                  */
1692                 if (oldstate != PATH_UP &&
1693                     oldstate != PATH_GHOST) {
1694                         if (pp->mpp->delay_watch_checks > 0)
1695                                 pp->watch_checks = pp->mpp->delay_watch_checks;
1696                         add_active = 1;
1697                 } else {
1698                         if (pp->watch_checks > 0)
1699                                 pp->watch_checks--;
1700                         add_active = 0;
1701                 }
1702                 if (!disable_reinstate && reinstate_path(pp, add_active)) {
1703                         condlog(3, "%s: reload map", pp->dev);
1704                         ev_add_path(pp, vecs, 1);
1705                         pp->tick = 1;
1706                         return 0;
1707                 }
1708                 new_path_up = 1;
1709
1710                 if (oldchkrstate != PATH_UP && oldchkrstate != PATH_GHOST)
1711                         chkr_new_path_up = 1;
1712
1713                 /*
1714                  * if at least one path is up in a group, and
1715                  * the group is disabled, re-enable it
1716                  */
1717                 if (newstate == PATH_UP)
1718                         enable_group(pp);
1719         }
1720         else if (newstate == PATH_UP || newstate == PATH_GHOST) {
1721                 if ((pp->dmstate == PSTATE_FAILED ||
1722                     pp->dmstate == PSTATE_UNDEF) &&
1723                     !disable_reinstate) {
1724                         /* Clear IO errors */
1725                         if (reinstate_path(pp, 0)) {
1726                                 condlog(3, "%s: reload map", pp->dev);
1727                                 ev_add_path(pp, vecs, 1);
1728                                 pp->tick = 1;
1729                                 return 0;
1730                         }
1731                 } else {
1732                         unsigned int max_checkint;
1733                         LOG_MSG(4, checker_message(&pp->checker));
1734                         conf = get_multipath_config();
1735                         max_checkint = conf->max_checkint;
1736                         put_multipath_config(conf);
1737                         if (pp->checkint != max_checkint) {
1738                                 /*
1739                                  * double the next check delay.
1740                                  * max at conf->max_checkint
1741                                  */
1742                                 if (pp->checkint < (max_checkint / 2))
1743                                         pp->checkint = 2 * pp->checkint;
1744                                 else
1745                                         pp->checkint = max_checkint;
1746
1747                                 condlog(4, "%s: delay next check %is",
1748                                         pp->dev_t, pp->checkint);
1749                         }
1750                         if (pp->watch_checks > 0)
1751                                 pp->watch_checks--;
1752                         pp->tick = pp->checkint;
1753                 }
1754         }
1755         else if (newstate != PATH_UP && newstate != PATH_GHOST) {
1756                 if (pp->dmstate == PSTATE_ACTIVE ||
1757                     pp->dmstate == PSTATE_UNDEF)
1758                         fail_path(pp, 0);
1759                 if (newstate == PATH_DOWN) {
1760                         int log_checker_err;
1761
1762                         conf = get_multipath_config();
1763                         log_checker_err = conf->log_checker_err;
1764                         put_multipath_config(conf);
1765                         if (log_checker_err == LOG_CHKR_ERR_ONCE)
1766                                 LOG_MSG(3, checker_message(&pp->checker));
1767                         else
1768                                 LOG_MSG(2, checker_message(&pp->checker));
1769                 }
1770         }
1771
1772         pp->state = newstate;
1773         repair_path(pp);
1774
1775         if (pp->mpp->wait_for_udev)
1776                 return 1;
1777         /*
1778          * path prio refreshing
1779          */
1780         condlog(4, "path prio refresh");
1781
1782         if (update_prio(pp, new_path_up) &&
1783             (pp->mpp->pgpolicyfn == (pgpolicyfn *)group_by_prio) &&
1784              pp->mpp->pgfailback == -FAILBACK_IMMEDIATE)
1785                 update_path_groups(pp->mpp, vecs, !new_path_up);
1786         else if (need_switch_pathgroup(pp->mpp, 0)) {
1787                 if (pp->mpp->pgfailback > 0 &&
1788                     (new_path_up || pp->mpp->failback_tick <= 0))
1789                         pp->mpp->failback_tick =
1790                                 pp->mpp->pgfailback + 1;
1791                 else if (pp->mpp->pgfailback == -FAILBACK_IMMEDIATE ||
1792                          (chkr_new_path_up && followover_should_failback(pp)))
1793                         switch_pathgroup(pp->mpp);
1794         }
1795         return 1;
1796 }
1797
1798 static void init_path_check_interval(struct vectors *vecs)
1799 {
1800         struct config *conf;
1801         struct path *pp;
1802         unsigned int i;
1803
1804         vector_foreach_slot (vecs->pathvec, pp, i) {
1805                 conf = get_multipath_config();
1806                 pp->checkint = conf->checkint;
1807                 put_multipath_config(conf);
1808         }
1809 }
1810
1811 static void *
1812 checkerloop (void *ap)
1813 {
1814         struct vectors *vecs;
1815         struct path *pp;
1816         int count = 0;
1817         unsigned int i;
1818         struct itimerval timer_tick_it;
1819         struct timespec last_time;
1820         struct config *conf;
1821
1822         pthread_cleanup_push(rcu_unregister, NULL);
1823         rcu_register_thread();
1824         mlockall(MCL_CURRENT | MCL_FUTURE);
1825         vecs = (struct vectors *)ap;
1826         condlog(2, "path checkers start up");
1827
1828         /* Tweak start time for initial path check */
1829         if (clock_gettime(CLOCK_MONOTONIC, &last_time) != 0)
1830                 last_time.tv_sec = 0;
1831         else
1832                 last_time.tv_sec -= 1;
1833
1834         while (1) {
1835                 struct timespec diff_time, start_time, end_time;
1836                 int num_paths = 0, ticks = 0, signo, strict_timing, rc = 0;
1837                 sigset_t mask;
1838
1839                 if (clock_gettime(CLOCK_MONOTONIC, &start_time) != 0)
1840                         start_time.tv_sec = 0;
1841                 if (start_time.tv_sec && last_time.tv_sec) {
1842                         timespecsub(&start_time, &last_time, &diff_time);
1843                         condlog(4, "tick (%lu.%06lu secs)",
1844                                 diff_time.tv_sec, diff_time.tv_nsec / 1000);
1845                         last_time = start_time;
1846                         ticks = diff_time.tv_sec;
1847                 } else {
1848                         ticks = 1;
1849                         condlog(4, "tick (%d ticks)", ticks);
1850                 }
1851 #ifdef USE_SYSTEMD
1852                 if (use_watchdog)
1853                         sd_notify(0, "WATCHDOG=1");
1854 #endif
1855                 rc = set_config_state(DAEMON_RUNNING);
1856                 if (rc == ETIMEDOUT) {
1857                         condlog(4, "timeout waiting for DAEMON_IDLE");
1858                         continue;
1859                 }
1860
1861                 pthread_cleanup_push(cleanup_lock, &vecs->lock);
1862                 lock(&vecs->lock);
1863                 pthread_testcancel();
1864                 vector_foreach_slot (vecs->pathvec, pp, i) {
1865                         rc = check_path(vecs, pp, ticks);
1866                         if (rc < 0) {
1867                                 vector_del_slot(vecs->pathvec, i);
1868                                 free_path(pp);
1869                                 i--;
1870                         } else
1871                                 num_paths += rc;
1872                 }
1873                 lock_cleanup_pop(vecs->lock);
1874
1875                 pthread_cleanup_push(cleanup_lock, &vecs->lock);
1876                 lock(&vecs->lock);
1877                 pthread_testcancel();
1878                 defered_failback_tick(vecs->mpvec);
1879                 retry_count_tick(vecs->mpvec);
1880                 missing_uev_wait_tick(vecs);
1881                 ghost_delay_tick(vecs);
1882                 lock_cleanup_pop(vecs->lock);
1883
1884                 if (count)
1885                         count--;
1886                 else {
1887                         pthread_cleanup_push(cleanup_lock, &vecs->lock);
1888                         lock(&vecs->lock);
1889                         pthread_testcancel();
1890                         condlog(4, "map garbage collection");
1891                         mpvec_garbage_collector(vecs);
1892                         count = MAPGCINT;
1893                         lock_cleanup_pop(vecs->lock);
1894                 }
1895
1896                 diff_time.tv_nsec = 0;
1897                 if (start_time.tv_sec &&
1898                     clock_gettime(CLOCK_MONOTONIC, &end_time) == 0) {
1899                         timespecsub(&end_time, &start_time, &diff_time);
1900                         if (num_paths) {
1901                                 unsigned int max_checkint;
1902
1903                                 condlog(3, "checked %d path%s in %lu.%06lu secs",
1904                                         num_paths, num_paths > 1 ? "s" : "",
1905                                         diff_time.tv_sec,
1906                                         diff_time.tv_nsec / 1000);
1907                                 conf = get_multipath_config();
1908                                 max_checkint = conf->max_checkint;
1909                                 put_multipath_config(conf);
1910                                 if (diff_time.tv_sec > max_checkint)
1911                                         condlog(1, "path checkers took longer "
1912                                                 "than %lu seconds, consider "
1913                                                 "increasing max_polling_interval",
1914                                                 diff_time.tv_sec);
1915                         }
1916                 }
1917
1918                 post_config_state(DAEMON_IDLE);
1919                 conf = get_multipath_config();
1920                 strict_timing = conf->strict_timing;
1921                 put_multipath_config(conf);
1922                 if (!strict_timing)
1923                         sleep(1);
1924                 else {
1925                         timer_tick_it.it_interval.tv_sec = 0;
1926                         timer_tick_it.it_interval.tv_usec = 0;
1927                         if (diff_time.tv_nsec) {
1928                                 timer_tick_it.it_value.tv_sec = 0;
1929                                 timer_tick_it.it_value.tv_usec =
1930                                      1000UL * 1000 * 1000 - diff_time.tv_nsec;
1931                         } else {
1932                                 timer_tick_it.it_value.tv_sec = 1;
1933                                 timer_tick_it.it_value.tv_usec = 0;
1934                         }
1935                         setitimer(ITIMER_REAL, &timer_tick_it, NULL);
1936
1937                         sigemptyset(&mask);
1938                         sigaddset(&mask, SIGALRM);
1939                         condlog(3, "waiting for %lu.%06lu secs",
1940                                 timer_tick_it.it_value.tv_sec,
1941                                 timer_tick_it.it_value.tv_usec);
1942                         if (sigwait(&mask, &signo) != 0) {
1943                                 condlog(3, "sigwait failed with error %d",
1944                                         errno);
1945                                 conf = get_multipath_config();
1946                                 conf->strict_timing = 0;
1947                                 put_multipath_config(conf);
1948                                 break;
1949                         }
1950                 }
1951         }
1952         pthread_cleanup_pop(1);
1953         return NULL;
1954 }
1955
1956 int
1957 configure (struct vectors * vecs)
1958 {
1959         struct multipath * mpp;
1960         struct path * pp;
1961         vector mpvec;
1962         int i, ret;
1963         struct config *conf;
1964         static int force_reload = FORCE_RELOAD_WEAK;
1965
1966         if (!vecs->pathvec && !(vecs->pathvec = vector_alloc())) {
1967                 condlog(0, "couldn't allocate path vec in configure");
1968                 return 1;
1969         }
1970
1971         if (!vecs->mpvec && !(vecs->mpvec = vector_alloc())) {
1972                 condlog(0, "couldn't allocate multipath vec in configure");
1973                 return 1;
1974         }
1975
1976         if (!(mpvec = vector_alloc())) {
1977                 condlog(0, "couldn't allocate new maps vec in configure");
1978                 return 1;
1979         }
1980
1981         /*
1982          * probe for current path (from sysfs) and map (from dm) sets
1983          */
1984         ret = path_discovery(vecs->pathvec, DI_ALL);
1985         if (ret < 0) {
1986                 condlog(0, "configure failed at path discovery");
1987                 return 1;
1988         }
1989
1990         vector_foreach_slot (vecs->pathvec, pp, i){
1991                 conf = get_multipath_config();
1992                 if (filter_path(conf, pp) > 0){
1993                         vector_del_slot(vecs->pathvec, i);
1994                         free_path(pp);
1995                         i--;
1996                 }
1997                 else
1998                         pp->checkint = conf->checkint;
1999                 put_multipath_config(conf);
2000         }
2001         if (map_discovery(vecs)) {
2002                 condlog(0, "configure failed at map discovery");
2003                 return 1;
2004         }
2005
2006         /*
2007          * create new set of maps & push changed ones into dm
2008          * In the first call, use FORCE_RELOAD_WEAK to avoid making
2009          * superfluous ACT_RELOAD ioctls. Later calls are done
2010          * with FORCE_RELOAD_YES.
2011          */
2012         ret = coalesce_paths(vecs, mpvec, NULL, force_reload, CMD_NONE);
2013         if (force_reload == FORCE_RELOAD_WEAK)
2014                 force_reload = FORCE_RELOAD_YES;
2015         if (ret) {
2016                 condlog(0, "configure failed while coalescing paths");
2017                 return 1;
2018         }
2019
2020         /*
2021          * may need to remove some maps which are no longer relevant
2022          * e.g., due to blacklist changes in conf file
2023          */
2024         if (coalesce_maps(vecs, mpvec)) {
2025                 condlog(0, "configure failed while coalescing maps");
2026                 return 1;
2027         }
2028
2029         dm_lib_release();
2030
2031         sync_maps_state(mpvec);
2032         vector_foreach_slot(mpvec, mpp, i){
2033                 remember_wwid(mpp->wwid);
2034                 update_map_pr(mpp);
2035         }
2036
2037         /*
2038          * purge dm of old maps
2039          */
2040         remove_maps(vecs);
2041
2042         /*
2043          * save new set of maps formed by considering current path state
2044          */
2045         vector_free(vecs->mpvec);
2046         vecs->mpvec = mpvec;
2047
2048         /*
2049          * start dm event waiter threads for these new maps
2050          */
2051         vector_foreach_slot(vecs->mpvec, mpp, i) {
2052                 if (setup_multipath(vecs, mpp)) {
2053                         i--;
2054                         continue;
2055                 }
2056                 if (start_waiter_thread(mpp, vecs)) {
2057                         remove_map(mpp, vecs, 1);
2058                         i--;
2059                 }
2060         }
2061         return 0;
2062 }
2063
2064 int
2065 need_to_delay_reconfig(struct vectors * vecs)
2066 {
2067         struct multipath *mpp;
2068         int i;
2069
2070         if (!VECTOR_SIZE(vecs->mpvec))
2071                 return 0;
2072
2073         vector_foreach_slot(vecs->mpvec, mpp, i) {
2074                 if (mpp->wait_for_udev)
2075                         return 1;
2076         }
2077         return 0;
2078 }
2079
2080 void rcu_free_config(struct rcu_head *head)
2081 {
2082         struct config *conf = container_of(head, struct config, rcu);
2083
2084         free_config(conf);
2085 }
2086
2087 int
2088 reconfigure (struct vectors * vecs)
2089 {
2090         struct config * old, *conf;
2091
2092         conf = load_config(DEFAULT_CONFIGFILE);
2093         if (!conf)
2094                 return 1;
2095
2096         /*
2097          * free old map and path vectors ... they use old conf state
2098          */
2099         if (VECTOR_SIZE(vecs->mpvec))
2100                 remove_maps_and_stop_waiters(vecs);
2101
2102         free_pathvec(vecs->pathvec, FREE_PATHS);
2103         vecs->pathvec = NULL;
2104
2105         /* Re-read any timezone changes */
2106         tzset();
2107
2108         dm_drv_version(conf->version, TGT_MPATH);
2109         if (verbosity)
2110                 conf->verbosity = verbosity;
2111         if (bindings_read_only)
2112                 conf->bindings_read_only = bindings_read_only;
2113         if (conf->find_multipaths) {
2114                 condlog(2, "find_multipaths is set: -n is implied");
2115                 ignore_new_devs = 1;
2116         }
2117         if (ignore_new_devs)
2118                 conf->ignore_new_devs = ignore_new_devs;
2119         uxsock_timeout = conf->uxsock_timeout;
2120
2121         old = rcu_dereference(multipath_conf);
2122         rcu_assign_pointer(multipath_conf, conf);
2123         call_rcu(&old->rcu, rcu_free_config);
2124
2125         configure(vecs);
2126
2127
2128         return 0;
2129 }
2130
2131 static struct vectors *
2132 init_vecs (void)
2133 {
2134         struct vectors * vecs;
2135
2136         vecs = (struct vectors *)MALLOC(sizeof(struct vectors));
2137
2138         if (!vecs)
2139                 return NULL;
2140
2141         pthread_mutex_init(&vecs->lock.mutex, NULL);
2142
2143         return vecs;
2144 }
2145
2146 static void *
2147 signal_set(int signo, void (*func) (int))
2148 {
2149         int r;
2150         struct sigaction sig;
2151         struct sigaction osig;
2152
2153         sig.sa_handler = func;
2154         sigemptyset(&sig.sa_mask);
2155         sig.sa_flags = 0;
2156
2157         r = sigaction(signo, &sig, &osig);
2158
2159         if (r < 0)
2160                 return (SIG_ERR);
2161         else
2162                 return (osig.sa_handler);
2163 }
2164
2165 void
2166 handle_signals(bool nonfatal)
2167 {
2168         if (exit_sig) {
2169                 condlog(2, "exit (signal)");
2170                 exit_sig = 0;
2171                 exit_daemon();
2172         }
2173         if (!nonfatal)
2174                 return;
2175         if (reconfig_sig) {
2176                 condlog(2, "reconfigure (signal)");
2177                 set_config_state(DAEMON_CONFIGURE);
2178         }
2179         if (log_reset_sig) {
2180                 condlog(2, "reset log (signal)");
2181                 pthread_mutex_lock(&logq_lock);
2182                 log_reset("multipathd");
2183                 pthread_mutex_unlock(&logq_lock);
2184         }
2185         reconfig_sig = 0;
2186         log_reset_sig = 0;
2187 }
2188
2189 static void
2190 sighup (int sig)
2191 {
2192         reconfig_sig = 1;
2193 }
2194
2195 static void
2196 sigend (int sig)
2197 {
2198         exit_sig = 1;
2199 }
2200
2201 static void
2202 sigusr1 (int sig)
2203 {
2204         log_reset_sig = 1;
2205 }
2206
2207 static void
2208 sigusr2 (int sig)
2209 {
2210         condlog(3, "SIGUSR2 received");
2211 }
2212
2213 static void
2214 signal_init(void)
2215 {
2216         sigset_t set;
2217
2218         sigemptyset(&set);
2219         sigaddset(&set, SIGUSR2);
2220         pthread_sigmask(SIG_SETMASK, &set, NULL);
2221
2222         signal_set(SIGHUP, sighup);
2223         signal_set(SIGUSR1, sigusr1);
2224         signal_set(SIGUSR2, sigusr2);
2225         signal_set(SIGINT, sigend);
2226         signal_set(SIGTERM, sigend);
2227         signal_set(SIGPIPE, sigend);
2228 }
2229
2230 static void
2231 setscheduler (void)
2232 {
2233         int res;
2234         static struct sched_param sched_param = {
2235                 .sched_priority = 99
2236         };
2237
2238         res = sched_setscheduler (0, SCHED_RR, &sched_param);
2239
2240         if (res == -1)
2241                 condlog(LOG_WARNING, "Could not set SCHED_RR at priority 99");
2242         return;
2243 }
2244
2245 static void
2246 set_oom_adj (void)
2247 {
2248 #ifdef OOM_SCORE_ADJ_MIN
2249         int retry = 1;
2250         char *file = "/proc/self/oom_score_adj";
2251         int score = OOM_SCORE_ADJ_MIN;
2252 #else
2253         int retry = 0;
2254         char *file = "/proc/self/oom_adj";
2255         int score = OOM_ADJUST_MIN;
2256 #endif
2257         FILE *fp;
2258         struct stat st;
2259         char *envp;
2260
2261         envp = getenv("OOMScoreAdjust");
2262         if (envp) {
2263                 condlog(3, "Using systemd provided OOMScoreAdjust");
2264                 return;
2265         }
2266         do {
2267                 if (stat(file, &st) == 0){
2268                         fp = fopen(file, "w");
2269                         if (!fp) {
2270                                 condlog(0, "couldn't fopen %s : %s", file,
2271                                         strerror(errno));
2272                                 return;
2273                         }
2274                         fprintf(fp, "%i", score);
2275                         fclose(fp);
2276                         return;
2277                 }
2278                 if (errno != ENOENT) {
2279                         condlog(0, "couldn't stat %s : %s", file,
2280                                 strerror(errno));
2281                         return;
2282                 }
2283 #ifdef OOM_ADJUST_MIN
2284                 file = "/proc/self/oom_adj";
2285                 score = OOM_ADJUST_MIN;
2286 #else
2287                 retry = 0;
2288 #endif
2289         } while (retry--);
2290         condlog(0, "couldn't adjust oom score");
2291 }
2292
2293 static int
2294 child (void * param)
2295 {
2296         pthread_t check_thr, uevent_thr, uxlsnr_thr, uevq_thr;
2297         pthread_attr_t log_attr, misc_attr, uevent_attr;
2298         struct vectors * vecs;
2299         struct multipath * mpp;
2300         int i;
2301 #ifdef USE_SYSTEMD
2302         unsigned long checkint;
2303         int startup_done = 0;
2304 #endif
2305         int rc;
2306         int pid_fd = -1;
2307         struct config *conf;
2308         char *envp;
2309
2310         mlockall(MCL_CURRENT | MCL_FUTURE);
2311         signal_init();
2312         rcu_init();
2313
2314         setup_thread_attr(&misc_attr, 64 * 1024, 0);
2315         setup_thread_attr(&uevent_attr, DEFAULT_UEVENT_STACKSIZE * 1024, 0);
2316         setup_thread_attr(&waiter_attr, 32 * 1024, 1);
2317         setup_thread_attr(&io_err_stat_attr, 32 * 1024, 0);
2318
2319         if (logsink == 1) {
2320                 setup_thread_attr(&log_attr, 64 * 1024, 0);
2321                 log_thread_start(&log_attr);
2322                 pthread_attr_destroy(&log_attr);
2323         }
2324         pid_fd = pidfile_create(DEFAULT_PIDFILE, daemon_pid);
2325         if (pid_fd < 0) {
2326                 condlog(1, "failed to create pidfile");
2327                 if (logsink == 1)
2328                         log_thread_stop();
2329                 exit(1);
2330         }
2331
2332         post_config_state(DAEMON_START);
2333
2334         condlog(2, "--------start up--------");
2335         condlog(2, "read " DEFAULT_CONFIGFILE);
2336
2337         conf = load_config(DEFAULT_CONFIGFILE);
2338         if (!conf)
2339                 goto failed;
2340
2341         if (verbosity)
2342                 conf->verbosity = verbosity;
2343         if (bindings_read_only)
2344                 conf->bindings_read_only = bindings_read_only;
2345         if (ignore_new_devs)
2346                 conf->ignore_new_devs = ignore_new_devs;
2347         uxsock_timeout = conf->uxsock_timeout;
2348         rcu_assign_pointer(multipath_conf, conf);
2349         if (init_checkers(conf->multipath_dir)) {
2350                 condlog(0, "failed to initialize checkers");
2351                 goto failed;
2352         }
2353         if (init_prio(conf->multipath_dir)) {
2354                 condlog(0, "failed to initialize prioritizers");
2355                 goto failed;
2356         }
2357
2358         setlogmask(LOG_UPTO(conf->verbosity + 3));
2359
2360         envp = getenv("LimitNOFILE");
2361
2362         if (envp) {
2363                 condlog(2,"Using systemd provided open fds limit of %s", envp);
2364         } else if (conf->max_fds) {
2365                 struct rlimit fd_limit;
2366
2367                 if (getrlimit(RLIMIT_NOFILE, &fd_limit) < 0) {
2368                         condlog(0, "can't get open fds limit: %s",
2369                                 strerror(errno));
2370                         fd_limit.rlim_cur = 0;
2371                         fd_limit.rlim_max = 0;
2372                 }
2373                 if (fd_limit.rlim_cur < conf->max_fds) {
2374                         fd_limit.rlim_cur = conf->max_fds;
2375                         if (fd_limit.rlim_max < conf->max_fds)
2376                                 fd_limit.rlim_max = conf->max_fds;
2377                         if (setrlimit(RLIMIT_NOFILE, &fd_limit) < 0) {
2378                                 condlog(0, "can't set open fds limit to "
2379                                         "%lu/%lu : %s",
2380                                         fd_limit.rlim_cur, fd_limit.rlim_max,
2381                                         strerror(errno));
2382                         } else {
2383                                 condlog(3, "set open fds limit to %lu/%lu",
2384                                         fd_limit.rlim_cur, fd_limit.rlim_max);
2385                         }
2386                 }
2387
2388         }
2389
2390         vecs = gvecs = init_vecs();
2391         if (!vecs)
2392                 goto failed;
2393
2394         setscheduler();
2395         set_oom_adj();
2396
2397 #ifdef USE_SYSTEMD
2398         envp = getenv("WATCHDOG_USEC");
2399         if (envp && sscanf(envp, "%lu", &checkint) == 1) {
2400                 /* Value is in microseconds */
2401                 conf->max_checkint = checkint / 1000000;
2402                 /* Rescale checkint */
2403                 if (conf->checkint > conf->max_checkint)
2404                         conf->checkint = conf->max_checkint;
2405                 else
2406                         conf->checkint = conf->max_checkint / 4;
2407                 condlog(3, "enabling watchdog, interval %d max %d",
2408                         conf->checkint, conf->max_checkint);
2409                 use_watchdog = conf->checkint;
2410         }
2411 #endif
2412         /*
2413          * Startup done, invalidate configuration
2414          */
2415         conf = NULL;
2416
2417         /*
2418          * Signal start of configuration
2419          */
2420         post_config_state(DAEMON_CONFIGURE);
2421
2422         init_path_check_interval(vecs);
2423
2424         /*
2425          * Start uevent listener early to catch events
2426          */
2427         if ((rc = pthread_create(&uevent_thr, &uevent_attr, ueventloop, udev))) {
2428                 condlog(0, "failed to create uevent thread: %d", rc);
2429                 goto failed;
2430         }
2431         pthread_attr_destroy(&uevent_attr);
2432         if ((rc = pthread_create(&uxlsnr_thr, &misc_attr, uxlsnrloop, vecs))) {
2433                 condlog(0, "failed to create cli listener: %d", rc);
2434                 goto failed;
2435         }
2436
2437         /*
2438          * start threads
2439          */
2440         rc = start_io_err_stat_thread(vecs);
2441         if (rc)
2442                 goto failed;
2443
2444         if ((rc = pthread_create(&check_thr, &misc_attr, checkerloop, vecs))) {
2445                 condlog(0,"failed to create checker loop thread: %d", rc);
2446                 goto failed;
2447         }
2448         if ((rc = pthread_create(&uevq_thr, &misc_attr, uevqloop, vecs))) {
2449                 condlog(0, "failed to create uevent dispatcher: %d", rc);
2450                 goto failed;
2451         }
2452         pthread_attr_destroy(&misc_attr);
2453
2454         while (running_state != DAEMON_SHUTDOWN) {
2455                 pthread_cleanup_push(config_cleanup, NULL);
2456                 pthread_mutex_lock(&config_lock);
2457                 if (running_state != DAEMON_CONFIGURE &&
2458                     running_state != DAEMON_SHUTDOWN) {
2459                         pthread_cond_wait(&config_cond, &config_lock);
2460                 }
2461                 pthread_cleanup_pop(1);
2462                 if (running_state == DAEMON_CONFIGURE) {
2463                         pthread_cleanup_push(cleanup_lock, &vecs->lock);
2464                         lock(&vecs->lock);
2465                         pthread_testcancel();
2466                         if (!need_to_delay_reconfig(vecs)) {
2467                                 reconfigure(vecs);
2468                         } else {
2469                                 conf = get_multipath_config();
2470                                 conf->delayed_reconfig = 1;
2471                                 put_multipath_config(conf);
2472                         }
2473                         lock_cleanup_pop(vecs->lock);
2474                         post_config_state(DAEMON_IDLE);
2475 #ifdef USE_SYSTEMD
2476                         if (!startup_done) {
2477                                 sd_notify(0, "READY=1");
2478                                 startup_done = 1;
2479                         }
2480 #endif
2481                 }
2482         }
2483
2484         lock(&vecs->lock);
2485         conf = get_multipath_config();
2486         if (conf->queue_without_daemon == QUE_NO_DAEMON_OFF)
2487                 vector_foreach_slot(vecs->mpvec, mpp, i)
2488                         dm_queue_if_no_path(mpp->alias, 0);
2489         put_multipath_config(conf);
2490         remove_maps_and_stop_waiters(vecs);
2491         unlock(&vecs->lock);
2492
2493         pthread_cancel(check_thr);
2494         pthread_cancel(uevent_thr);
2495         pthread_cancel(uxlsnr_thr);
2496         pthread_cancel(uevq_thr);
2497
2498         pthread_join(check_thr, NULL);
2499         pthread_join(uevent_thr, NULL);
2500         pthread_join(uxlsnr_thr, NULL);
2501         pthread_join(uevq_thr, NULL);
2502
2503         stop_io_err_stat_thread();
2504
2505         lock(&vecs->lock);
2506         free_pathvec(vecs->pathvec, FREE_PATHS);
2507         vecs->pathvec = NULL;
2508         unlock(&vecs->lock);
2509
2510         pthread_mutex_destroy(&vecs->lock.mutex);
2511         FREE(vecs);
2512         vecs = NULL;
2513
2514         cleanup_checkers();
2515         cleanup_prio();
2516
2517         dm_lib_release();
2518         dm_lib_exit();
2519
2520         /* We're done here */
2521         condlog(3, "unlink pidfile");
2522         unlink(DEFAULT_PIDFILE);
2523
2524         condlog(2, "--------shut down-------");
2525
2526         if (logsink == 1)
2527                 log_thread_stop();
2528
2529         /*
2530          * Freeing config must be done after condlog() and dm_lib_exit(),
2531          * because logging functions like dlog() and dm_write_log()
2532          * reference the config.
2533          */
2534         conf = rcu_dereference(multipath_conf);
2535         rcu_assign_pointer(multipath_conf, NULL);
2536         call_rcu(&conf->rcu, rcu_free_config);
2537         udev_unref(udev);
2538         udev = NULL;
2539         pthread_attr_destroy(&waiter_attr);
2540         pthread_attr_destroy(&io_err_stat_attr);
2541 #ifdef _DEBUG_
2542         dbg_free_final(NULL);
2543 #endif
2544
2545 #ifdef USE_SYSTEMD
2546         sd_notify(0, "ERRNO=0");
2547 #endif
2548         exit(0);
2549
2550 failed:
2551 #ifdef USE_SYSTEMD
2552         sd_notify(0, "ERRNO=1");
2553 #endif
2554         if (pid_fd >= 0)
2555                 close(pid_fd);
2556         exit(1);
2557 }
2558
2559 static int
2560 daemonize(void)
2561 {
2562         int pid;
2563         int dev_null_fd;
2564
2565         if( (pid = fork()) < 0){
2566                 fprintf(stderr, "Failed first fork : %s\n", strerror(errno));
2567                 return -1;
2568         }
2569         else if (pid != 0)
2570                 return pid;
2571
2572         setsid();
2573
2574         if ( (pid = fork()) < 0)
2575                 fprintf(stderr, "Failed second fork : %s\n", strerror(errno));
2576         else if (pid != 0)
2577                 _exit(0);
2578
2579         if (chdir("/") < 0)
2580                 fprintf(stderr, "cannot chdir to '/', continuing\n");
2581
2582         dev_null_fd = open("/dev/null", O_RDWR);
2583         if (dev_null_fd < 0){
2584                 fprintf(stderr, "cannot open /dev/null for input & output : %s\n",
2585                         strerror(errno));
2586                 _exit(0);
2587         }
2588
2589         close(STDIN_FILENO);
2590         if (dup(dev_null_fd) < 0) {
2591                 fprintf(stderr, "cannot dup /dev/null to stdin : %s\n",
2592                         strerror(errno));
2593                 _exit(0);
2594         }
2595         close(STDOUT_FILENO);
2596         if (dup(dev_null_fd) < 0) {
2597                 fprintf(stderr, "cannot dup /dev/null to stdout : %s\n",
2598                         strerror(errno));
2599                 _exit(0);
2600         }
2601         close(STDERR_FILENO);
2602         if (dup(dev_null_fd) < 0) {
2603                 fprintf(stderr, "cannot dup /dev/null to stderr : %s\n",
2604                         strerror(errno));
2605                 _exit(0);
2606         }
2607         close(dev_null_fd);
2608         daemon_pid = getpid();
2609         return 0;
2610 }
2611
2612 int
2613 main (int argc, char *argv[])
2614 {
2615         extern char *optarg;
2616         extern int optind;
2617         int arg;
2618         int err;
2619         int foreground = 0;
2620         struct config *conf;
2621
2622         ANNOTATE_BENIGN_RACE_SIZED(&multipath_conf, sizeof(multipath_conf),
2623                                    "Manipulated through RCU");
2624         ANNOTATE_BENIGN_RACE_SIZED(&running_state, sizeof(running_state),
2625                 "Suppress complaints about unprotected running_state reads");
2626         ANNOTATE_BENIGN_RACE_SIZED(&uxsock_timeout, sizeof(uxsock_timeout),
2627                 "Suppress complaints about this scalar variable");
2628
2629         logsink = 1;
2630
2631         if (getuid() != 0) {
2632                 fprintf(stderr, "need to be root\n");
2633                 exit(1);
2634         }
2635
2636         /* make sure we don't lock any path */
2637         if (chdir("/") < 0)
2638                 fprintf(stderr, "can't chdir to root directory : %s\n",
2639                         strerror(errno));
2640         umask(umask(077) | 022);
2641
2642         pthread_cond_init_mono(&config_cond);
2643
2644         udev = udev_new();
2645         libmp_udev_set_sync_support(0);
2646
2647         while ((arg = getopt(argc, argv, ":dsv:k::Bn")) != EOF ) {
2648                 switch(arg) {
2649                 case 'd':
2650                         foreground = 1;
2651                         if (logsink > 0)
2652                                 logsink = 0;
2653                         //debug=1; /* ### comment me out ### */
2654                         break;
2655                 case 'v':
2656                         if (sizeof(optarg) > sizeof(char *) ||
2657                             !isdigit(optarg[0]))
2658                                 exit(1);
2659
2660                         verbosity = atoi(optarg);
2661                         break;
2662                 case 's':
2663                         logsink = -1;
2664                         break;
2665                 case 'k':
2666                         conf = load_config(DEFAULT_CONFIGFILE);
2667                         if (!conf)
2668                                 exit(1);
2669                         if (verbosity)
2670                                 conf->verbosity = verbosity;
2671                         uxsock_timeout = conf->uxsock_timeout;
2672                         uxclnt(optarg, uxsock_timeout + 100);
2673                         free_config(conf);
2674                         exit(0);
2675                 case 'B':
2676                         bindings_read_only = 1;
2677                         break;
2678                 case 'n':
2679                         ignore_new_devs = 1;
2680                         break;
2681                 default:
2682                         fprintf(stderr, "Invalid argument '-%c'\n",
2683                                 optopt);
2684                         exit(1);
2685                 }
2686         }
2687         if (optind < argc) {
2688                 char cmd[CMDSIZE];
2689                 char * s = cmd;
2690                 char * c = s;
2691
2692                 conf = load_config(DEFAULT_CONFIGFILE);
2693                 if (!conf)
2694                         exit(1);
2695                 if (verbosity)
2696                         conf->verbosity = verbosity;
2697                 uxsock_timeout = conf->uxsock_timeout;
2698                 memset(cmd, 0x0, CMDSIZE);
2699                 while (optind < argc) {
2700                         if (strchr(argv[optind], ' '))
2701                                 c += snprintf(c, s + CMDSIZE - c, "\"%s\" ", argv[optind]);
2702                         else
2703                                 c += snprintf(c, s + CMDSIZE - c, "%s ", argv[optind]);
2704                         optind++;
2705                 }
2706                 c += snprintf(c, s + CMDSIZE - c, "\n");
2707                 uxclnt(s, uxsock_timeout + 100);
2708                 free_config(conf);
2709                 exit(0);
2710         }
2711
2712         if (foreground) {
2713                 if (!isatty(fileno(stdout)))
2714                         setbuf(stdout, NULL);
2715                 err = 0;
2716                 daemon_pid = getpid();
2717         } else
2718                 err = daemonize();
2719
2720         if (err < 0)
2721                 /* error */
2722                 exit(1);
2723         else if (err > 0)
2724                 /* parent dies */
2725                 exit(0);
2726         else
2727                 /* child lives */
2728                 return (child(NULL));
2729 }
2730
2731 void *  mpath_pr_event_handler_fn (void * pathp )
2732 {
2733         struct multipath * mpp;
2734         int i, ret, isFound;
2735         struct path * pp = (struct path *)pathp;
2736         struct prout_param_descriptor *param;
2737         struct prin_resp *resp;
2738
2739         mpp = pp->mpp;
2740
2741         resp = mpath_alloc_prin_response(MPATH_PRIN_RKEY_SA);
2742         if (!resp){
2743                 condlog(0,"%s Alloc failed for prin response", pp->dev);
2744                 return NULL;
2745         }
2746
2747         ret = prin_do_scsi_ioctl(pp->dev, MPATH_PRIN_RKEY_SA, resp, 0);
2748         if (ret != MPATH_PR_SUCCESS )
2749         {
2750                 condlog(0,"%s : pr in read keys service action failed. Error=%d", pp->dev, ret);
2751                 goto out;
2752         }
2753
2754         condlog(3, " event pr=%d addlen=%d",resp->prin_descriptor.prin_readkeys.prgeneration,
2755                         resp->prin_descriptor.prin_readkeys.additional_length );
2756
2757         if (resp->prin_descriptor.prin_readkeys.additional_length == 0 )
2758         {
2759                 condlog(1, "%s: No key found. Device may not be registered.", pp->dev);
2760                 ret = MPATH_PR_SUCCESS;
2761                 goto out;
2762         }
2763         condlog(2, "Multipath  reservation_key: 0x%" PRIx64 " ",
2764                 get_be64(mpp->reservation_key));
2765
2766         isFound =0;
2767         for (i = 0; i < resp->prin_descriptor.prin_readkeys.additional_length/8; i++ )
2768         {
2769                 condlog(2, "PR IN READKEYS[%d]  reservation key:",i);
2770                 dumpHex((char *)&resp->prin_descriptor.prin_readkeys.key_list[i*8], 8 , -1);
2771                 if (!memcmp(&mpp->reservation_key, &resp->prin_descriptor.prin_readkeys.key_list[i*8], 8))
2772                 {
2773                         condlog(2, "%s: pr key found in prin readkeys response", mpp->alias);
2774                         isFound =1;
2775                         break;
2776                 }
2777         }
2778         if (!isFound)
2779         {
2780                 condlog(0, "%s: Either device not registered or ", pp->dev);
2781                 condlog(0, "host is not authorised for registration. Skip path");
2782                 ret = MPATH_PR_OTHER;
2783                 goto out;
2784         }
2785
2786         param= malloc(sizeof(struct prout_param_descriptor));
2787         memset(param, 0 , sizeof(struct prout_param_descriptor));
2788         memcpy(param->sa_key, &mpp->reservation_key, 8);
2789         param->num_transportid = 0;
2790
2791         condlog(3, "device %s:%s", pp->dev, pp->mpp->wwid);
2792
2793         ret = prout_do_scsi_ioctl(pp->dev, MPATH_PROUT_REG_IGN_SA, 0, 0, param, 0);
2794         if (ret != MPATH_PR_SUCCESS )
2795         {
2796                 condlog(0,"%s: Reservation registration failed. Error: %d", pp->dev, ret);
2797         }
2798         mpp->prflag = 1;
2799
2800         free(param);
2801 out:
2802         free(resp);
2803         return NULL;
2804 }
2805
2806 int mpath_pr_event_handle(struct path *pp)
2807 {
2808         pthread_t thread;
2809         int rc;
2810         pthread_attr_t attr;
2811         struct multipath * mpp;
2812
2813         mpp = pp->mpp;
2814
2815         if (get_be64(mpp->reservation_key))
2816                 return -1;
2817
2818         pthread_attr_init(&attr);
2819         pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
2820
2821         rc = pthread_create(&thread, NULL , mpath_pr_event_handler_fn, pp);
2822         if (rc) {
2823                 condlog(0, "%s: ERROR; return code from pthread_create() is %d", pp->dev, rc);
2824                 return -1;
2825         }
2826         pthread_attr_destroy(&attr);
2827         rc = pthread_join(thread, NULL);
2828         return 0;
2829 }