2e79667df08ba802269828dd899e6e8e9b6ee74f
[multipath-tools/.git] / libmultipath / devmapper.c
1 /*
2  * snippets copied from device-mapper dmsetup.c
3  * Copyright (c) 2004, 2005 Christophe Varoqui
4  * Copyright (c) 2005 Kiyoshi Ueda, NEC
5  * Copyright (c) 2005 Patrick Caulfield, Redhat
6  */
7 #include <stdio.h>
8 #include <stdlib.h>
9 #include <stdarg.h>
10 #include <string.h>
11 #include <libdevmapper.h>
12 #include <ctype.h>
13 #include <unistd.h>
14 #include <errno.h>
15 #include <sys/sysmacros.h>
16
17 #include "checkers.h"
18 #include "vector.h"
19 #include "structs.h"
20 #include "debug.h"
21 #include "memory.h"
22 #include "devmapper.h"
23 #include "sysfs.h"
24 #include "config.h"
25 #include "wwids.h"
26
27 #include "log_pthread.h"
28 #include <sys/types.h>
29 #include <time.h>
30
31 #define MAX_WAIT 5
32 #define LOOPS_PER_SEC 5
33
34 static int dm_conf_verbosity;
35
36 #ifdef LIBDM_API_DEFERRED
37 static int dm_cancel_remove_partmaps(const char * mapname);
38 #endif
39
40 static int do_foreach_partmaps(const char * mapname,
41                                int (*partmap_func)(const char *, void *),
42                                void *data);
43
44 #ifndef LIBDM_API_COOKIE
45 static inline int dm_task_set_cookie(struct dm_task *dmt, uint32_t *c, int a)
46 {
47         return 1;
48 }
49
50 void dm_udev_wait(unsigned int c)
51 {
52 }
53
54 void dm_udev_set_sync_support(int c)
55 {
56 }
57
58 #endif
59
60 static void
61 dm_write_log (int level, const char *file, int line, const char *f, ...)
62 {
63         va_list ap;
64         int thres;
65
66         if (level > 6)
67                 level = 6;
68
69         thres = dm_conf_verbosity;
70         if (thres <= 3 || level > thres)
71                 return;
72
73         va_start(ap, f);
74         if (logsink < 1) {
75                 if (logsink == 0) {
76                         time_t t = time(NULL);
77                         struct tm *tb = localtime(&t);
78                         char buff[16];
79
80                         strftime(buff, sizeof(buff), "%b %d %H:%M:%S", tb);
81                         buff[sizeof(buff)-1] = '\0';
82
83                         fprintf(stderr, "%s | ", buff);
84                 }
85                 fprintf(stderr, "libdevmapper: %s(%i): ", file, line);
86                 vfprintf(stderr, f, ap);
87                 fprintf(stderr, "\n");
88         } else {
89                 condlog(level, "libdevmapper: %s(%i): ", file, line);
90                 log_safe(level + 3, f, ap);
91         }
92         va_end(ap);
93
94         return;
95 }
96
97 void dm_init(int v)
98 {
99         dm_conf_verbosity = v;
100         dm_log_init(&dm_write_log);
101         dm_log_init_verbose(v + 3);
102 }
103
104 static int
105 dm_lib_prereq (void)
106 {
107         char version[64];
108         int v[3];
109 #if defined(LIBDM_API_DEFERRED)
110         int minv[3] = {1, 2, 89};
111 #elif defined(DM_SUBSYSTEM_UDEV_FLAG0)
112         int minv[3] = {1, 2, 82};
113 #elif defined(LIBDM_API_COOKIE)
114         int minv[3] = {1, 2, 38};
115 #else
116         int minv[3] = {1, 2, 8};
117 #endif
118
119         dm_get_library_version(version, sizeof(version));
120         condlog(3, "libdevmapper version %s", version);
121         if (sscanf(version, "%d.%d.%d ", &v[0], &v[1], &v[2]) != 3) {
122                 condlog(0, "invalid libdevmapper version %s", version);
123                 return 1;
124         }
125
126         if VERSION_GE(v, minv)
127                 return 0;
128         condlog(0, "libdevmapper version must be >= %d.%.2d.%.2d",
129                 minv[0], minv[1], minv[2]);
130         return 1;
131 }
132
133 int
134 dm_drv_version(unsigned int *v)
135 {
136         char buff[64];
137
138         v[0] = 0;
139         v[1] = 0;
140         v[2] = 0;
141
142         if (!dm_driver_version(buff, sizeof(buff))) {
143                 condlog(0, "cannot get kernel dm version");
144                 return 1;
145         }
146         if (sscanf(buff, "%u.%u.%u ", &v[0], &v[1], &v[2]) != 3) {
147                 condlog(0, "invalid kernel dm version '%s'", buff);
148                 return 1;
149         }
150         return 0;
151 }
152
153 int
154 dm_tgt_version (unsigned int * version, char * str)
155 {
156         int r = 2;
157         struct dm_task *dmt;
158         struct dm_versions *target;
159         struct dm_versions *last_target;
160         unsigned int *v;
161
162         version[0] = 0;
163         version[1] = 0;
164         version[2] = 0;
165
166         if (!(dmt = dm_task_create(DM_DEVICE_LIST_VERSIONS)))
167                 return 1;
168
169         dm_task_no_open_count(dmt);
170
171         if (!dm_task_run(dmt)) {
172                 condlog(0, "Can not communicate with kernel DM");
173                 goto out;
174         }
175         target = dm_task_get_versions(dmt);
176
177         do {
178                 last_target = target;
179                 if (!strncmp(str, target->name, strlen(str))) {
180                         r = 1;
181                         break;
182                 }
183                 target = (void *) target + target->next;
184         } while (last_target != target);
185
186         if (r == 2) {
187                 condlog(0, "DM %s kernel driver not loaded", str);
188                 goto out;
189         }
190         v = target->version;
191         version[0] = v[0];
192         version[1] = v[1];
193         version[2] = v[2];
194         r = 0;
195 out:
196         dm_task_destroy(dmt);
197         return r;
198 }
199
200 static int
201 dm_tgt_prereq (unsigned int *ver)
202 {
203         unsigned int minv[3] = {1, 0, 3};
204         unsigned int version[3] = {0, 0, 0};
205         unsigned int * v = version;
206
207         if (dm_tgt_version(v, TGT_MPATH)) {
208                 /* in doubt return not capable */
209                 return 1;
210         }
211
212         /* test request based multipath capability */
213         condlog(3, "DM multipath kernel driver v%u.%u.%u",
214                 v[0], v[1], v[2]);
215
216         if (VERSION_GE(v, minv)) {
217                 ver[0] = v[0];
218                 ver[1] = v[1];
219                 ver[2] = v[2];
220                 return 0;
221         }
222
223         condlog(0, "DM multipath kernel driver must be >= v%u.%u.%u",
224                 minv[0], minv[1], minv[2]);
225         return 1;
226 }
227
228 static int dm_prereq(unsigned int *v)
229 {
230         if (dm_lib_prereq())
231                 return 1;
232         return dm_tgt_prereq(v);
233 }
234
235 static int libmp_dm_udev_sync = 0;
236
237 void libmp_udev_set_sync_support(int on)
238 {
239         libmp_dm_udev_sync = !!on;
240 }
241
242 void libmp_dm_init(void)
243 {
244         struct config *conf;
245         int verbosity;
246         unsigned int version[3];
247
248         if (dm_prereq(version))
249                 exit(1);
250         conf = get_multipath_config();
251         verbosity = conf->verbosity;
252         memcpy(conf->version, version, sizeof(version));
253         put_multipath_config(conf);
254         dm_init(verbosity);
255         dm_udev_set_sync_support(libmp_dm_udev_sync);
256 }
257
258 struct dm_task*
259 libmp_dm_task_create(int task)
260 {
261         static pthread_once_t dm_initialized = PTHREAD_ONCE_INIT;
262
263         pthread_once(&dm_initialized, libmp_dm_init);
264         return dm_task_create(task);
265 }
266
267 #define do_deferred(x) ((x) == DEFERRED_REMOVE_ON || (x) == DEFERRED_REMOVE_IN_PROGRESS)
268
269 static int
270 dm_simplecmd (int task, const char *name, int no_flush, int need_sync, uint16_t udev_flags, int deferred_remove) {
271         int r = 0;
272         int udev_wait_flag = ((need_sync || udev_flags) &&
273                               (task == DM_DEVICE_RESUME ||
274                                task == DM_DEVICE_REMOVE));
275         uint32_t cookie = 0;
276         struct dm_task *dmt;
277
278         if (!(dmt = libmp_dm_task_create (task)))
279                 return 0;
280
281         if (!dm_task_set_name (dmt, name))
282                 goto out;
283
284         dm_task_no_open_count(dmt);
285         dm_task_skip_lockfs(dmt);       /* for DM_DEVICE_RESUME */
286 #ifdef LIBDM_API_FLUSH
287         if (no_flush)
288                 dm_task_no_flush(dmt);          /* for DM_DEVICE_SUSPEND/RESUME */
289 #endif
290 #ifdef LIBDM_API_DEFERRED
291         if (do_deferred(deferred_remove))
292                 dm_task_deferred_remove(dmt);
293 #endif
294         if (udev_wait_flag &&
295             !dm_task_set_cookie(dmt, &cookie,
296                                 DM_UDEV_DISABLE_LIBRARY_FALLBACK | udev_flags))
297                 goto out;
298
299         r = dm_task_run (dmt);
300
301         if (udev_wait_flag)
302                         dm_udev_wait(cookie);
303 out:
304         dm_task_destroy (dmt);
305         return r;
306 }
307
308 int dm_simplecmd_flush (int task, const char *name, uint16_t udev_flags)
309 {
310         return dm_simplecmd(task, name, 0, 1, udev_flags, 0);
311 }
312
313 int dm_simplecmd_noflush (int task, const char *name, uint16_t udev_flags)
314 {
315         return dm_simplecmd(task, name, 1, 1, udev_flags, 0);
316 }
317
318 static int
319 dm_device_remove (const char *name, int needsync, int deferred_remove) {
320         return dm_simplecmd(DM_DEVICE_REMOVE, name, 0, needsync, 0,
321                             deferred_remove);
322 }
323
324 static int
325 dm_addmap (int task, const char *target, struct multipath *mpp,
326            char * params, int ro, uint16_t udev_flags) {
327         int r = 0;
328         struct dm_task *dmt;
329         char *prefixed_uuid = NULL;
330         uint32_t cookie = 0;
331
332         /* Need to add this here to allow 0 to be passed in udev_flags */
333         udev_flags |= DM_UDEV_DISABLE_LIBRARY_FALLBACK;
334
335         if (!(dmt = libmp_dm_task_create (task)))
336                 return 0;
337
338         if (!dm_task_set_name (dmt, mpp->alias))
339                 goto addout;
340
341         if (!dm_task_add_target (dmt, 0, mpp->size, target, params))
342                 goto addout;
343
344         if (ro)
345                 dm_task_set_ro(dmt);
346
347         if (task == DM_DEVICE_CREATE) {
348                 if (strlen(mpp->wwid) > 0) {
349                         prefixed_uuid = MALLOC(UUID_PREFIX_LEN +
350                                                strlen(mpp->wwid) + 1);
351                         if (!prefixed_uuid) {
352                                 condlog(0, "cannot create prefixed uuid : %s",
353                                         strerror(errno));
354                                 goto addout;
355                         }
356                         sprintf(prefixed_uuid, UUID_PREFIX "%s", mpp->wwid);
357                         if (!dm_task_set_uuid(dmt, prefixed_uuid))
358                                 goto freeout;
359                 }
360                 dm_task_skip_lockfs(dmt);
361 #ifdef LIBDM_API_FLUSH
362                 dm_task_no_flush(dmt);
363 #endif
364         }
365
366         if (mpp->attribute_flags & (1 << ATTR_MODE) &&
367             !dm_task_set_mode(dmt, mpp->mode))
368                 goto freeout;
369         if (mpp->attribute_flags & (1 << ATTR_UID) &&
370             !dm_task_set_uid(dmt, mpp->uid))
371                 goto freeout;
372         if (mpp->attribute_flags & (1 << ATTR_GID) &&
373             !dm_task_set_gid(dmt, mpp->gid))
374                 goto freeout;
375         condlog(4, "%s: %s [0 %llu %s %s]", mpp->alias,
376                 task == DM_DEVICE_RELOAD ? "reload" : "addmap", mpp->size,
377                 target, params);
378
379         dm_task_no_open_count(dmt);
380
381         if (task == DM_DEVICE_CREATE &&
382             !dm_task_set_cookie(dmt, &cookie, udev_flags))
383                 goto freeout;
384
385         r = dm_task_run (dmt);
386
387         if (task == DM_DEVICE_CREATE)
388                         dm_udev_wait(cookie);
389 freeout:
390         if (prefixed_uuid)
391                 FREE(prefixed_uuid);
392
393 addout:
394         dm_task_destroy (dmt);
395
396         return r;
397 }
398
399 static uint16_t build_udev_flags(const struct multipath *mpp, int reload)
400 {
401         /* DM_UDEV_DISABLE_LIBRARY_FALLBACK is added in dm_addmap */
402         return  (mpp->skip_kpartx == SKIP_KPARTX_ON ?
403                  MPATH_UDEV_NO_KPARTX_FLAG : 0) |
404                 ((mpp->nr_active == 0 || mpp->ghost_delay_tick > 0)?
405                  MPATH_UDEV_NO_PATHS_FLAG : 0) |
406                 (reload && !mpp->force_udev_reload ?
407                  MPATH_UDEV_RELOAD_FLAG : 0);
408 }
409
410 int dm_addmap_create (struct multipath *mpp, char * params)
411 {
412         int ro;
413         uint16_t udev_flags = build_udev_flags(mpp, 0);
414
415         for (ro = 0; ro <= 1; ro++) {
416                 int err;
417
418                 if (dm_addmap(DM_DEVICE_CREATE, TGT_MPATH, mpp, params, ro,
419                               udev_flags)) {
420                         if (unmark_failed_wwid(mpp->wwid) ==
421                             WWID_FAILED_CHANGED)
422                                 mpp->needs_paths_uevent = 1;
423                         return 1;
424                 }
425                 /*
426                  * DM_DEVICE_CREATE is actually DM_DEV_CREATE + DM_TABLE_LOAD.
427                  * Failing the second part leaves an empty map. Clean it up.
428                  */
429                 err = errno;
430                 if (dm_map_present(mpp->alias)) {
431                         condlog(3, "%s: failed to load map (a path might be in use)", mpp->alias);
432                         dm_flush_map_nosync(mpp->alias);
433                 }
434                 if (err != EROFS) {
435                         condlog(3, "%s: failed to load map, error %d",
436                                 mpp->alias, err);
437                         break;
438                 }
439         }
440         if (mark_failed_wwid(mpp->wwid) == WWID_FAILED_CHANGED)
441                 mpp->needs_paths_uevent = 1;
442         return 0;
443 }
444
445 #define ADDMAP_RW 0
446 #define ADDMAP_RO 1
447
448 int dm_addmap_reload(struct multipath *mpp, char *params, int flush)
449 {
450         int r = 0;
451         uint16_t udev_flags = build_udev_flags(mpp, 1);
452
453         /*
454          * DM_DEVICE_RELOAD cannot wait on a cookie, as
455          * the cookie will only ever be released after an
456          * DM_DEVICE_RESUME. So call DM_DEVICE_RESUME
457          * after each successful call to DM_DEVICE_RELOAD.
458          */
459         if (!mpp->force_readonly)
460                 r = dm_addmap(DM_DEVICE_RELOAD, TGT_MPATH, mpp, params,
461                               ADDMAP_RW, 0);
462         if (!r) {
463                 if (!mpp->force_readonly && errno != EROFS)
464                         return 0;
465                 r = dm_addmap(DM_DEVICE_RELOAD, TGT_MPATH, mpp,
466                               params, ADDMAP_RO, 0);
467         }
468         if (r)
469                 r = dm_simplecmd(DM_DEVICE_RESUME, mpp->alias, !flush,
470                                  1, udev_flags, 0);
471         if (r)
472                 return r;
473
474         /* If the resume failed, dm will leave the device suspended, and
475          * drop the new table, so doing a second resume will try using
476          * the original table */
477         if (dm_is_suspended(mpp->alias))
478                 dm_simplecmd(DM_DEVICE_RESUME, mpp->alias, !flush, 1,
479                              udev_flags, 0);
480         return 0;
481 }
482
483 static int
484 do_get_info(const char *name, struct dm_info *info)
485 {
486         int r = -1;
487         struct dm_task *dmt;
488
489         if (!(dmt = libmp_dm_task_create(DM_DEVICE_INFO)))
490                 return r;
491
492         if (!dm_task_set_name(dmt, name))
493                 goto out;
494
495         dm_task_no_open_count(dmt);
496
497         if (!dm_task_run(dmt))
498                 goto out;
499
500         if (!dm_task_get_info(dmt, info))
501                 goto out;
502
503         if (!info->exists)
504                 goto out;
505
506         r = 0;
507 out:
508         dm_task_destroy(dmt);
509         return r;
510 }
511
512 int dm_map_present(const char * str)
513 {
514         struct dm_info info;
515
516         return (do_get_info(str, &info) == 0);
517 }
518
519 int dm_get_map(const char *name, unsigned long long *size, char *outparams)
520 {
521         int r = 1;
522         struct dm_task *dmt;
523         uint64_t start, length;
524         char *target_type = NULL;
525         char *params = NULL;
526
527         if (!(dmt = libmp_dm_task_create(DM_DEVICE_TABLE)))
528                 return 1;
529
530         if (!dm_task_set_name(dmt, name))
531                 goto out;
532
533         dm_task_no_open_count(dmt);
534
535         if (!dm_task_run(dmt))
536                 goto out;
537
538         /* Fetch 1st target */
539         dm_get_next_target(dmt, NULL, &start, &length,
540                            &target_type, &params);
541
542         if (size)
543                 *size = length;
544
545         if (!outparams) {
546                 r = 0;
547                 goto out;
548         }
549         if (snprintf(outparams, PARAMS_SIZE, "%s", params) <= PARAMS_SIZE)
550                 r = 0;
551 out:
552         dm_task_destroy(dmt);
553         return r;
554 }
555
556 static int
557 dm_get_prefixed_uuid(const char *name, char *uuid)
558 {
559         struct dm_task *dmt;
560         const char *uuidtmp;
561         int r = 1;
562
563         dmt = libmp_dm_task_create(DM_DEVICE_INFO);
564         if (!dmt)
565                 return 1;
566
567         if (!dm_task_set_name (dmt, name))
568                 goto uuidout;
569
570         if (!dm_task_run(dmt))
571                 goto uuidout;
572
573         uuidtmp = dm_task_get_uuid(dmt);
574         if (uuidtmp)
575                 strcpy(uuid, uuidtmp);
576         else
577                 uuid[0] = '\0';
578
579         r = 0;
580 uuidout:
581         dm_task_destroy(dmt);
582         return r;
583 }
584
585 int dm_get_uuid(const char *name, char *uuid)
586 {
587         if (dm_get_prefixed_uuid(name, uuid))
588                 return 1;
589
590         if (!strncmp(uuid, UUID_PREFIX, UUID_PREFIX_LEN))
591                 memmove(uuid, uuid + UUID_PREFIX_LEN,
592                         strlen(uuid + UUID_PREFIX_LEN) + 1);
593         return 0;
594 }
595
596 static int
597 is_mpath_part(const char *part_name, const char *map_name)
598 {
599         char *p;
600         char part_uuid[WWID_SIZE], map_uuid[WWID_SIZE];
601
602         if (dm_get_prefixed_uuid(part_name, part_uuid))
603                 return 0;
604
605         if (dm_get_prefixed_uuid(map_name, map_uuid))
606                 return 0;
607
608         if (strncmp(part_uuid, "part", 4) != 0)
609                 return 0;
610
611         p = strstr(part_uuid, UUID_PREFIX);
612         if (p && !strcmp(p, map_uuid))
613                 return 1;
614
615         return 0;
616 }
617
618 int dm_get_status(const char *name, char *outstatus)
619 {
620         int r = 1;
621         struct dm_task *dmt;
622         uint64_t start, length;
623         char *target_type = NULL;
624         char *status = NULL;
625
626         if (!(dmt = libmp_dm_task_create(DM_DEVICE_STATUS)))
627                 return 1;
628
629         if (!dm_task_set_name(dmt, name))
630                 goto out;
631
632         dm_task_no_open_count(dmt);
633
634         if (!dm_task_run(dmt))
635                 goto out;
636
637         /* Fetch 1st target */
638         dm_get_next_target(dmt, NULL, &start, &length,
639                            &target_type, &status);
640         if (!status) {
641                 condlog(2, "get null status.");
642                 goto out;
643         }
644
645         if (snprintf(outstatus, PARAMS_SIZE, "%s", status) <= PARAMS_SIZE)
646                 r = 0;
647 out:
648         if (r)
649                 condlog(0, "%s: error getting map status string", name);
650
651         dm_task_destroy(dmt);
652         return r;
653 }
654
655 /*
656  * returns:
657  *    1 : match
658  *    0 : no match
659  *   -1 : empty map, or more than 1 target
660  */
661 int dm_type(const char *name, char *type)
662 {
663         int r = 0;
664         struct dm_task *dmt;
665         uint64_t start, length;
666         char *target_type = NULL;
667         char *params;
668
669         if (!(dmt = libmp_dm_task_create(DM_DEVICE_TABLE)))
670                 return 0;
671
672         if (!dm_task_set_name(dmt, name))
673                 goto out;
674
675         dm_task_no_open_count(dmt);
676
677         if (!dm_task_run(dmt))
678                 goto out;
679
680         /* Fetch 1st target */
681         if (dm_get_next_target(dmt, NULL, &start, &length,
682                                &target_type, &params) != NULL)
683                 /* multiple targets */
684                 r = -1;
685         else if (!target_type)
686                 r = -1;
687         else if (!strcmp(target_type, type))
688                 r = 1;
689
690 out:
691         dm_task_destroy(dmt);
692         return r;
693 }
694
695 /*
696  * returns:
697  * 1  : is multipath device
698  * 0  : is not multipath device
699  * -1 : error
700  */
701 int dm_is_mpath(const char *name)
702 {
703         int r = -1;
704         struct dm_task *dmt;
705         struct dm_info info;
706         uint64_t start, length;
707         char *target_type = NULL;
708         char *params;
709         const char *uuid;
710
711         if (!(dmt = libmp_dm_task_create(DM_DEVICE_TABLE)))
712                 goto out;
713
714         if (!dm_task_set_name(dmt, name))
715                 goto out_task;
716
717         dm_task_no_open_count(dmt);
718
719         if (!dm_task_run(dmt))
720                 goto out_task;
721
722         if (!dm_task_get_info(dmt, &info))
723                 goto out_task;
724
725         r = 0;
726
727         if (!info.exists)
728                 goto out_task;
729
730         uuid = dm_task_get_uuid(dmt);
731
732         if (!uuid || strncmp(uuid, UUID_PREFIX, UUID_PREFIX_LEN) != 0)
733                 goto out_task;
734
735         /* Fetch 1st target */
736         if (dm_get_next_target(dmt, NULL, &start, &length, &target_type,
737                                &params) != NULL)
738                 /* multiple targets */
739                 goto out_task;
740
741         if (!target_type || strcmp(target_type, TGT_MPATH) != 0)
742                 goto out_task;
743
744         r = 1;
745 out_task:
746         dm_task_destroy(dmt);
747 out:
748         if (r < 0)
749                 condlog(3, "%s: dm command failed in %s: %s", name, __FUNCTION__, strerror(errno));
750         return r;
751 }
752
753 static int
754 dm_dev_t (const char * mapname, char * dev_t, int len)
755 {
756         struct dm_info info;
757
758         if (do_get_info(mapname, &info) != 0)
759                 return 1;
760
761         if (snprintf(dev_t, len, "%i:%i", info.major, info.minor) > len)
762                 return 1;
763
764         return 0;
765 }
766
767 int
768 dm_get_opencount (const char * mapname)
769 {
770         int r = -1;
771         struct dm_task *dmt;
772         struct dm_info info;
773
774         if (!(dmt = libmp_dm_task_create(DM_DEVICE_INFO)))
775                 return 0;
776
777         if (!dm_task_set_name(dmt, mapname))
778                 goto out;
779
780         if (!dm_task_run(dmt))
781                 goto out;
782
783         if (!dm_task_get_info(dmt, &info))
784                 goto out;
785
786         if (!info.exists)
787                 goto out;
788
789         r = info.open_count;
790 out:
791         dm_task_destroy(dmt);
792         return r;
793 }
794
795 int
796 dm_get_major_minor(const char *name, int *major, int *minor)
797 {
798         struct dm_info info;
799
800         if (do_get_info(name, &info) != 0)
801                 return -1;
802
803         *major = info.major;
804         *minor = info.minor;
805         return 0;
806 }
807
808 static int
809 has_partmap(const char *name, void *data)
810 {
811         return 1;
812 }
813
814 static int
815 partmap_in_use(const char *name, void *data)
816 {
817         int part_count, *ret_count = (int *)data;
818         int open_count = dm_get_opencount(name);
819
820         if (ret_count)
821                 (*ret_count)++;
822         part_count = 0;
823         if (open_count) {
824                 if (do_foreach_partmaps(name, partmap_in_use, &part_count))
825                         return 1;
826                 if (open_count != part_count) {
827                         condlog(2, "%s: map in use", name);
828                         return 1;
829                 }
830         }
831         return 0;
832 }
833
834 int _dm_flush_map (const char * mapname, int need_sync, int deferred_remove,
835                    int need_suspend, int retries)
836 {
837         int r;
838         int queue_if_no_path = 0;
839         int udev_flags = 0;
840         unsigned long long mapsize;
841         char params[PARAMS_SIZE] = {0};
842
843         if (dm_is_mpath(mapname) != 1)
844                 return 0; /* nothing to do */
845
846         /* if the device currently has no partitions, do not
847            run kpartx on it if you fail to delete it */
848         if (do_foreach_partmaps(mapname, has_partmap, NULL) == 0)
849                 udev_flags |= MPATH_UDEV_NO_KPARTX_FLAG;
850
851         /* If you aren't doing a deferred remove, make sure that no
852          * devices are in use */
853         if (!do_deferred(deferred_remove) && partmap_in_use(mapname, NULL))
854                         return 1;
855
856         if (need_suspend &&
857             !dm_get_map(mapname, &mapsize, params) &&
858             strstr(params, "queue_if_no_path")) {
859                 if (!dm_queue_if_no_path(mapname, 0))
860                         queue_if_no_path = 1;
861                 else
862                         /* Leave queue_if_no_path alone if unset failed */
863                         queue_if_no_path = -1;
864         }
865
866         if (dm_remove_partmaps(mapname, need_sync, deferred_remove))
867                 return 1;
868
869         if (!do_deferred(deferred_remove) && dm_get_opencount(mapname)) {
870                 condlog(2, "%s: map in use", mapname);
871                 return 1;
872         }
873
874         do {
875                 if (need_suspend && queue_if_no_path != -1)
876                         dm_simplecmd_flush(DM_DEVICE_SUSPEND, mapname, 0);
877
878                 r = dm_device_remove(mapname, need_sync, deferred_remove);
879
880                 if (r) {
881                         if (do_deferred(deferred_remove)
882                             && dm_map_present(mapname)) {
883                                 condlog(4, "multipath map %s remove deferred",
884                                         mapname);
885                                 return 2;
886                         }
887                         condlog(4, "multipath map %s removed", mapname);
888                         return 0;
889                 } else {
890                         condlog(2, "failed to remove multipath map %s",
891                                 mapname);
892                         if (need_suspend && queue_if_no_path != -1) {
893                                 dm_simplecmd_noflush(DM_DEVICE_RESUME,
894                                                      mapname, udev_flags);
895                         }
896                 }
897                 if (retries)
898                         sleep(1);
899         } while (retries-- > 0);
900
901         if (queue_if_no_path == 1)
902                 dm_queue_if_no_path(mapname, 1);
903
904         return 1;
905 }
906
907 #ifdef LIBDM_API_DEFERRED
908
909 int
910 dm_flush_map_nopaths(const char * mapname, int deferred_remove)
911 {
912         return _dm_flush_map(mapname, 1, deferred_remove, 0, 0);
913 }
914
915 #else
916
917 int
918 dm_flush_map_nopaths(const char * mapname, int deferred_remove)
919 {
920         return _dm_flush_map(mapname, 1, 0, 0, 0);
921 }
922
923 #endif
924
925 int dm_flush_maps (int retries)
926 {
927         int r = 0;
928         struct dm_task *dmt;
929         struct dm_names *names;
930         unsigned next = 0;
931
932         if (!(dmt = libmp_dm_task_create (DM_DEVICE_LIST)))
933                 return 0;
934
935         dm_task_no_open_count(dmt);
936
937         if (!dm_task_run (dmt))
938                 goto out;
939
940         if (!(names = dm_task_get_names (dmt)))
941                 goto out;
942
943         if (!names->dev)
944                 goto out;
945
946         do {
947                 r |= dm_suspend_and_flush_map(names->name, retries);
948                 next = names->next;
949                 names = (void *) names + next;
950         } while (next);
951
952 out:
953         dm_task_destroy (dmt);
954         return r;
955 }
956
957 int
958 dm_message(const char * mapname, char * message)
959 {
960         int r = 1;
961         struct dm_task *dmt;
962
963         if (!(dmt = libmp_dm_task_create(DM_DEVICE_TARGET_MSG)))
964                 return 1;
965
966         if (!dm_task_set_name(dmt, mapname))
967                 goto out;
968
969         if (!dm_task_set_sector(dmt, 0))
970                 goto out;
971
972         if (!dm_task_set_message(dmt, message))
973                 goto out;
974
975         dm_task_no_open_count(dmt);
976
977         if (!dm_task_run(dmt))
978                 goto out;
979
980         r = 0;
981 out:
982         if (r)
983                 condlog(0, "DM message failed [%s]", message);
984
985         dm_task_destroy(dmt);
986         return r;
987 }
988
989 int
990 dm_fail_path(const char * mapname, char * path)
991 {
992         char message[32];
993
994         if (snprintf(message, 32, "fail_path %s", path) > 32)
995                 return 1;
996
997         return dm_message(mapname, message);
998 }
999
1000 int
1001 dm_reinstate_path(const char * mapname, char * path)
1002 {
1003         char message[32];
1004
1005         if (snprintf(message, 32, "reinstate_path %s", path) > 32)
1006                 return 1;
1007
1008         return dm_message(mapname, message);
1009 }
1010
1011 int
1012 dm_queue_if_no_path(const char *mapname, int enable)
1013 {
1014         char *message;
1015
1016         if (enable)
1017                 message = "queue_if_no_path";
1018         else
1019                 message = "fail_if_no_path";
1020
1021         return dm_message(mapname, message);
1022 }
1023
1024 static int
1025 dm_groupmsg (const char * msg, const char * mapname, int index)
1026 {
1027         char message[32];
1028
1029         if (snprintf(message, 32, "%s_group %i", msg, index) > 32)
1030                 return 1;
1031
1032         return dm_message(mapname, message);
1033 }
1034
1035 int
1036 dm_switchgroup(const char * mapname, int index)
1037 {
1038         return dm_groupmsg("switch", mapname, index);
1039 }
1040
1041 int
1042 dm_enablegroup(const char * mapname, int index)
1043 {
1044         return dm_groupmsg("enable", mapname, index);
1045 }
1046
1047 int
1048 dm_disablegroup(const char * mapname, int index)
1049 {
1050         return dm_groupmsg("disable", mapname, index);
1051 }
1052
1053 struct multipath *dm_get_multipath(const char *name)
1054 {
1055         struct multipath *mpp = NULL;
1056
1057         mpp = alloc_multipath();
1058         if (!mpp)
1059                 return NULL;
1060
1061         mpp->alias = STRDUP(name);
1062
1063         if (!mpp->alias)
1064                 goto out;
1065
1066         if (dm_get_map(name, &mpp->size, NULL))
1067                 goto out;
1068
1069         dm_get_uuid(name, mpp->wwid);
1070         dm_get_info(name, &mpp->dmi);
1071
1072         return mpp;
1073 out:
1074         free_multipath(mpp, KEEP_PATHS);
1075         return NULL;
1076 }
1077
1078 int
1079 dm_get_maps (vector mp)
1080 {
1081         struct multipath * mpp;
1082         int r = 1;
1083         struct dm_task *dmt;
1084         struct dm_names *names;
1085         unsigned next = 0;
1086
1087         if (!mp)
1088                 return 1;
1089
1090         if (!(dmt = libmp_dm_task_create(DM_DEVICE_LIST)))
1091                 return 1;
1092
1093         dm_task_no_open_count(dmt);
1094
1095         if (!dm_task_run(dmt))
1096                 goto out;
1097
1098         if (!(names = dm_task_get_names(dmt)))
1099                 goto out;
1100
1101         if (!names->dev) {
1102                 r = 0; /* this is perfectly valid */
1103                 goto out;
1104         }
1105
1106         do {
1107                 if (dm_is_mpath(names->name) != 1)
1108                         goto next;
1109
1110                 mpp = dm_get_multipath(names->name);
1111                 if (!mpp)
1112                         goto out;
1113
1114                 if (!vector_alloc_slot(mp))
1115                         goto out;
1116
1117                 vector_set_slot(mp, mpp);
1118                 mpp = NULL;
1119 next:
1120                 next = names->next;
1121                 names = (void *) names + next;
1122         } while (next);
1123
1124         r = 0;
1125         goto out;
1126 out:
1127         dm_task_destroy (dmt);
1128         return r;
1129 }
1130
1131 int
1132 dm_geteventnr (const char *name)
1133 {
1134         struct dm_info info;
1135
1136         if (do_get_info(name, &info) != 0)
1137                 return -1;
1138
1139         return info.event_nr;
1140 }
1141
1142 int
1143 dm_is_suspended(const char *name)
1144 {
1145         struct dm_info info;
1146
1147         if (do_get_info(name, &info) != 0)
1148                 return -1;
1149
1150         return info.suspended;
1151 }
1152
1153 char *
1154 dm_mapname(int major, int minor)
1155 {
1156         char * response = NULL;
1157         const char *map;
1158         struct dm_task *dmt;
1159         int r;
1160         int loop = MAX_WAIT * LOOPS_PER_SEC;
1161
1162         if (!(dmt = libmp_dm_task_create(DM_DEVICE_STATUS)))
1163                 return NULL;
1164
1165         if (!dm_task_set_major(dmt, major) ||
1166             !dm_task_set_minor(dmt, minor))
1167                 goto bad;
1168
1169         dm_task_no_open_count(dmt);
1170
1171         /*
1172          * device map might not be ready when we get here from
1173          * daemon uev_trigger -> uev_add_map
1174          */
1175         while (--loop) {
1176                 r = dm_task_run(dmt);
1177
1178                 if (r)
1179                         break;
1180
1181                 usleep(1000 * 1000 / LOOPS_PER_SEC);
1182         }
1183
1184         if (!r) {
1185                 condlog(0, "%i:%i: timeout fetching map name", major, minor);
1186                 goto bad;
1187         }
1188
1189         map = dm_task_get_name(dmt);
1190         if (map && strlen(map))
1191                 response = STRDUP((const char *)dm_task_get_name(dmt));
1192
1193         dm_task_destroy(dmt);
1194         return response;
1195 bad:
1196         dm_task_destroy(dmt);
1197         condlog(0, "%i:%i: error fetching map name", major, minor);
1198         return NULL;
1199 }
1200
1201 static int
1202 do_foreach_partmaps (const char * mapname,
1203                      int (*partmap_func)(const char *, void *),
1204                      void *data)
1205 {
1206         struct dm_task *dmt;
1207         struct dm_names *names;
1208         unsigned next = 0;
1209         char params[PARAMS_SIZE];
1210         unsigned long long size;
1211         char dev_t[32];
1212         int r = 1;
1213         char *p;
1214
1215         if (!(dmt = libmp_dm_task_create(DM_DEVICE_LIST)))
1216                 return 1;
1217
1218         dm_task_no_open_count(dmt);
1219
1220         if (!dm_task_run(dmt))
1221                 goto out;
1222
1223         if (!(names = dm_task_get_names(dmt)))
1224                 goto out;
1225
1226         if (!names->dev) {
1227                 r = 0; /* this is perfectly valid */
1228                 goto out;
1229         }
1230
1231         if (dm_dev_t(mapname, &dev_t[0], 32))
1232                 goto out;
1233
1234         do {
1235                 if (
1236                     /*
1237                      * if there is only a single "linear" target
1238                      */
1239                     (dm_type(names->name, TGT_PART) == 1) &&
1240
1241                     /*
1242                      * and the uuid of the target is a partition of the
1243                      * uuid of the multipath device
1244                      */
1245                     is_mpath_part(names->name, mapname) &&
1246
1247                     /*
1248                      * and we can fetch the map table from the kernel
1249                      */
1250                     !dm_get_map(names->name, &size, &params[0]) &&
1251
1252                     /*
1253                      * and the table maps over the multipath map
1254                      */
1255                     (p = strstr(params, dev_t)) &&
1256                     !isdigit(*(p + strlen(dev_t)))
1257                    ) {
1258                         if (partmap_func(names->name, data) != 0)
1259                                 goto out;
1260                 }
1261
1262                 next = names->next;
1263                 names = (void *) names + next;
1264         } while (next);
1265
1266         r = 0;
1267 out:
1268         dm_task_destroy (dmt);
1269         return r;
1270 }
1271
1272 struct remove_data {
1273         int need_sync;
1274         int deferred_remove;
1275 };
1276
1277 static int
1278 remove_partmap(const char *name, void *data)
1279 {
1280         struct remove_data *rd = (struct remove_data *)data;
1281
1282         if (dm_get_opencount(name)) {
1283                 dm_remove_partmaps(name, rd->need_sync, rd->deferred_remove);
1284                 if (!do_deferred(rd->deferred_remove) &&
1285                     dm_get_opencount(name)) {
1286                         condlog(2, "%s: map in use", name);
1287                         return 1;
1288                 }
1289         }
1290         condlog(4, "partition map %s removed", name);
1291         dm_device_remove(name, rd->need_sync, rd->deferred_remove);
1292         return 0;
1293 }
1294
1295 int
1296 dm_remove_partmaps (const char * mapname, int need_sync, int deferred_remove)
1297 {
1298         struct remove_data rd = { need_sync, deferred_remove };
1299         return do_foreach_partmaps(mapname, remove_partmap, &rd);
1300 }
1301
1302 #ifdef LIBDM_API_DEFERRED
1303
1304 static int
1305 cancel_remove_partmap (const char *name, void *unused)
1306 {
1307         if (dm_get_opencount(name))
1308                 dm_cancel_remove_partmaps(name);
1309         if (dm_message(name, "@cancel_deferred_remove") != 0)
1310                 condlog(0, "%s: can't cancel deferred remove: %s", name,
1311                         strerror(errno));
1312         return 0;
1313 }
1314
1315 static int
1316 dm_get_deferred_remove (const char * mapname)
1317 {
1318         struct dm_info info;
1319
1320         if (do_get_info(mapname, &info) != 0)
1321                 return -1;
1322
1323         return info.deferred_remove;
1324 }
1325
1326 static int
1327 dm_cancel_remove_partmaps(const char * mapname) {
1328         return do_foreach_partmaps(mapname, cancel_remove_partmap, NULL);
1329 }
1330
1331 int
1332 dm_cancel_deferred_remove (struct multipath *mpp)
1333 {
1334         int r = 0;
1335
1336         if (!dm_get_deferred_remove(mpp->alias))
1337                 return 0;
1338         if (mpp->deferred_remove == DEFERRED_REMOVE_IN_PROGRESS)
1339                 mpp->deferred_remove = DEFERRED_REMOVE_ON;
1340
1341         dm_cancel_remove_partmaps(mpp->alias);
1342         r = dm_message(mpp->alias, "@cancel_deferred_remove");
1343         if (r)
1344                 condlog(0, "%s: can't cancel deferred remove: %s", mpp->alias,
1345                                 strerror(errno));
1346         else
1347                 condlog(2, "%s: canceled deferred remove", mpp->alias);
1348         return r;
1349 }
1350
1351 #else
1352
1353 int
1354 dm_cancel_deferred_remove (struct multipath *mpp)
1355 {
1356         return 0;
1357 }
1358
1359 #endif
1360
1361 static struct dm_info *
1362 alloc_dminfo (void)
1363 {
1364         return MALLOC(sizeof(struct dm_info));
1365 }
1366
1367 int
1368 dm_get_info (const char * mapname, struct dm_info ** dmi)
1369 {
1370         if (!mapname)
1371                 return 1;
1372
1373         if (!*dmi)
1374                 *dmi = alloc_dminfo();
1375
1376         if (!*dmi)
1377                 return 1;
1378
1379         if (do_get_info(mapname, *dmi) != 0) {
1380                 memset(*dmi, 0, sizeof(struct dm_info));
1381                 FREE(*dmi);
1382                 *dmi = NULL;
1383                 return 1;
1384         }
1385         return 0;
1386 }
1387
1388 struct rename_data {
1389         const char *old;
1390         char *new;
1391         char *delim;
1392 };
1393
1394 static int
1395 rename_partmap (const char *name, void *data)
1396 {
1397         char buff[PARAMS_SIZE];
1398         int offset;
1399         struct rename_data *rd = (struct rename_data *)data;
1400
1401         if (strncmp(name, rd->old, strlen(rd->old)) != 0)
1402                 return 0;
1403         for (offset = strlen(rd->old); name[offset] && !(isdigit(name[offset])); offset++); /* do nothing */
1404         snprintf(buff, PARAMS_SIZE, "%s%s%s", rd->new, rd->delim,
1405                  name + offset);
1406         dm_rename(name, buff, rd->delim, SKIP_KPARTX_OFF);
1407         condlog(4, "partition map %s renamed", name);
1408         return 0;
1409 }
1410
1411 int
1412 dm_rename_partmaps (const char * old, char * new, char *delim)
1413 {
1414         struct rename_data rd;
1415
1416         rd.old = old;
1417         rd.new = new;
1418
1419         if (delim)
1420                 rd.delim = delim;
1421         else {
1422                 if (isdigit(new[strlen(new)-1]))
1423                         rd.delim = "p";
1424                 else
1425                         rd.delim = "";
1426         }
1427         return do_foreach_partmaps(old, rename_partmap, &rd);
1428 }
1429
1430 int
1431 dm_rename (const char * old, char * new, char *delim, int skip_kpartx)
1432 {
1433         int r = 0;
1434         struct dm_task *dmt;
1435         uint32_t cookie = 0;
1436         uint16_t udev_flags = DM_UDEV_DISABLE_LIBRARY_FALLBACK | ((skip_kpartx == SKIP_KPARTX_ON)? MPATH_UDEV_NO_KPARTX_FLAG : 0);
1437
1438         if (dm_rename_partmaps(old, new, delim))
1439                 return r;
1440
1441         if (!(dmt = libmp_dm_task_create(DM_DEVICE_RENAME)))
1442                 return r;
1443
1444         if (!dm_task_set_name(dmt, old))
1445                 goto out;
1446
1447         if (!dm_task_set_newname(dmt, new))
1448                 goto out;
1449
1450         dm_task_no_open_count(dmt);
1451
1452         if (!dm_task_set_cookie(dmt, &cookie, udev_flags))
1453                 goto out;
1454         r = dm_task_run(dmt);
1455
1456         dm_udev_wait(cookie);
1457
1458 out:
1459         dm_task_destroy(dmt);
1460
1461         return r;
1462 }
1463
1464 void dm_reassign_deps(char *table, const char *dep, const char *newdep)
1465 {
1466         char *n, *newtable;
1467         const char *p;
1468
1469         newtable = strdup(table);
1470         if (!newtable)
1471                 return;
1472         p = strstr(newtable, dep);
1473         n = table + (p - newtable);
1474         strcpy(n, newdep);
1475         n += strlen(newdep);
1476         p += strlen(dep);
1477         strcat(n, p);
1478         FREE(newtable);
1479 }
1480
1481 int dm_reassign_table(const char *name, char *old, char *new)
1482 {
1483         int r = 0, modified = 0;
1484         uint64_t start, length;
1485         struct dm_task *dmt, *reload_dmt;
1486         char *target, *params = NULL;
1487         char *buff;
1488         void *next = NULL;
1489
1490         if (!(dmt = libmp_dm_task_create(DM_DEVICE_TABLE)))
1491                 return 0;
1492
1493         if (!dm_task_set_name(dmt, name))
1494                 goto out;
1495
1496         dm_task_no_open_count(dmt);
1497
1498         if (!dm_task_run(dmt))
1499                 goto out;
1500         if (!(reload_dmt = libmp_dm_task_create(DM_DEVICE_RELOAD)))
1501                 goto out;
1502         if (!dm_task_set_name(reload_dmt, name))
1503                 goto out_reload;
1504
1505         do {
1506                 next = dm_get_next_target(dmt, next, &start, &length,
1507                                           &target, &params);
1508                 buff = strdup(params);
1509                 if (!buff) {
1510                         condlog(3, "%s: failed to replace target %s, "
1511                                 "out of memory", name, target);
1512                         goto out_reload;
1513                 }
1514                 if (strcmp(target, TGT_MPATH) && strstr(params, old)) {
1515                         condlog(3, "%s: replace target %s %s",
1516                                 name, target, buff);
1517                         dm_reassign_deps(buff, old, new);
1518                         condlog(3, "%s: with target %s %s",
1519                                 name, target, buff);
1520                         modified++;
1521                 }
1522                 dm_task_add_target(reload_dmt, start, length, target, buff);
1523                 free(buff);
1524         } while (next);
1525
1526         if (modified) {
1527                 dm_task_no_open_count(reload_dmt);
1528
1529                 if (!dm_task_run(reload_dmt)) {
1530                         condlog(3, "%s: failed to reassign targets", name);
1531                         goto out_reload;
1532                 }
1533                 dm_simplecmd_noflush(DM_DEVICE_RESUME, name,
1534                                      MPATH_UDEV_RELOAD_FLAG);
1535         }
1536         r = 1;
1537
1538 out_reload:
1539         dm_task_destroy(reload_dmt);
1540 out:
1541         dm_task_destroy(dmt);
1542         return r;
1543 }
1544
1545
1546 /*
1547  * Reassign existing device-mapper table(s) to not use
1548  * the block devices but point to the multipathed
1549  * device instead
1550  */
1551 int dm_reassign(const char *mapname)
1552 {
1553         struct dm_deps *deps;
1554         struct dm_task *dmt;
1555         struct dm_info info;
1556         char dev_t[32], dm_dep[32];
1557         int r = 0, i;
1558
1559         if (dm_dev_t(mapname, &dev_t[0], 32)) {
1560                 condlog(3, "%s: failed to get device number", mapname);
1561                 return 1;
1562         }
1563
1564         if (!(dmt = libmp_dm_task_create(DM_DEVICE_DEPS))) {
1565                 condlog(3, "%s: couldn't make dm task", mapname);
1566                 return 0;
1567         }
1568
1569         if (!dm_task_set_name(dmt, mapname))
1570                 goto out;
1571
1572         dm_task_no_open_count(dmt);
1573
1574         if (!dm_task_run(dmt))
1575                 goto out;
1576
1577         if (!dm_task_get_info(dmt, &info))
1578                 goto out;
1579
1580         if (!(deps = dm_task_get_deps(dmt)))
1581                 goto out;
1582
1583         if (!info.exists)
1584                 goto out;
1585
1586         for (i = 0; i < deps->count; i++) {
1587                 sprintf(dm_dep, "%d:%d",
1588                         major(deps->device[i]),
1589                         minor(deps->device[i]));
1590                 sysfs_check_holders(dm_dep, dev_t);
1591         }
1592
1593         r = 1;
1594 out:
1595         dm_task_destroy (dmt);
1596         return r;
1597 }
1598
1599 int dm_setgeometry(struct multipath *mpp)
1600 {
1601         struct dm_task *dmt;
1602         struct path *pp;
1603         char heads[4], sectors[4];
1604         char cylinders[10], start[32];
1605         int r = 0;
1606
1607         if (!mpp)
1608                 return 1;
1609
1610         pp = first_path(mpp);
1611         if (!pp) {
1612                 condlog(3, "%s: no path for geometry", mpp->alias);
1613                 return 1;
1614         }
1615         if (pp->geom.cylinders == 0 ||
1616             pp->geom.heads == 0 ||
1617             pp->geom.sectors == 0) {
1618                 condlog(3, "%s: invalid geometry on %s", mpp->alias, pp->dev);
1619                 return 1;
1620         }
1621
1622         if (!(dmt = libmp_dm_task_create(DM_DEVICE_SET_GEOMETRY)))
1623                 return 0;
1624
1625         if (!dm_task_set_name(dmt, mpp->alias))
1626                 goto out;
1627
1628         dm_task_no_open_count(dmt);
1629
1630         /* What a sick interface ... */
1631         snprintf(heads, 4, "%u", pp->geom.heads);
1632         snprintf(sectors, 4, "%u", pp->geom.sectors);
1633         snprintf(cylinders, 10, "%u", pp->geom.cylinders);
1634         snprintf(start, 32, "%lu", pp->geom.start);
1635         if (!dm_task_set_geometry(dmt, cylinders, heads, sectors, start)) {
1636                 condlog(3, "%s: Failed to set geometry", mpp->alias);
1637                 goto out;
1638         }
1639
1640         r = dm_task_run(dmt);
1641 out:
1642         dm_task_destroy(dmt);
1643
1644         return r;
1645 }