75a6df67c207d07dd2301c8c425c25a5508b3e84
[multipath-tools/.git] / libmultipath / io_err_stat.c
1 /*
2  * (C) Copyright HUAWEI Technology Corp. 2017, All Rights Reserved.
3  *
4  * io_err_stat.c
5  * version 1.0
6  *
7  * IO error stream statistic process for path failure event from kernel
8  *
9  * Author(s): Guan Junxiong 2017 <guanjunxiong@huawei.com>
10  *
11  * This file is released under the GPL version 2, or any later version.
12  */
13
14 #include <unistd.h>
15 #include <pthread.h>
16 #include <signal.h>
17 #include <fcntl.h>
18 #include <sys/stat.h>
19 #include <sys/ioctl.h>
20 #include <linux/fs.h>
21 #include <libaio.h>
22 #include <errno.h>
23 #include <sys/mman.h>
24
25 #include "vector.h"
26 #include "memory.h"
27 #include "checkers.h"
28 #include "config.h"
29 #include "structs.h"
30 #include "structs_vec.h"
31 #include "devmapper.h"
32 #include "debug.h"
33 #include "lock.h"
34 #include "time-util.h"
35 #include "io_err_stat.h"
36
37 #define IOTIMEOUT_SEC                   60
38 #define TIMEOUT_NO_IO_NSEC              10000000 /*10ms = 10000000ns*/
39 #define FLAKY_PATHFAIL_THRESHOLD        2
40 #define CONCUR_NR_EVENT                 32
41
42 #define PATH_IO_ERR_IN_CHECKING         -1
43 #define PATH_IO_ERR_IN_POLLING_RECHECK  -2
44
45 #define io_err_stat_log(prio, fmt, args...) \
46         condlog(prio, "io error statistic: " fmt, ##args)
47
48
49 struct io_err_stat_pathvec {
50         pthread_mutex_t mutex;
51         vector          pathvec;
52 };
53
54 struct dio_ctx {
55         struct timespec io_starttime;
56         int             blksize;
57         void            *buf;
58         struct iocb     io;
59 };
60
61 struct io_err_stat_path {
62         char            devname[FILE_NAME_SIZE];
63         int             fd;
64         struct dio_ctx  *dio_ctx_array;
65         int             io_err_nr;
66         int             io_nr;
67         struct timespec start_time;
68
69         int             total_time;
70         int             err_rate_threshold;
71 };
72
73 pthread_t               io_err_stat_thr;
74 pthread_attr_t          io_err_stat_attr;
75
76 static struct io_err_stat_pathvec *paths;
77 struct vectors *vecs;
78 io_context_t    ioctx;
79
80 static void cancel_inflight_io(struct io_err_stat_path *pp);
81
82 static void rcu_unregister(void *param)
83 {
84         rcu_unregister_thread();
85 }
86
87 struct io_err_stat_path *find_err_path_by_dev(vector pathvec, char *dev)
88 {
89         int i;
90         struct io_err_stat_path *pp;
91
92         if (!pathvec)
93                 return NULL;
94         vector_foreach_slot(pathvec, pp, i)
95                 if (!strcmp(pp->devname, dev))
96                         return pp;
97
98         io_err_stat_log(4, "%s: not found in check queue", dev);
99
100         return NULL;
101 }
102
103 static int init_each_dio_ctx(struct dio_ctx *ct, int blksize,
104                 unsigned long pgsize)
105 {
106         ct->blksize = blksize;
107         if (posix_memalign(&ct->buf, pgsize, blksize))
108                 return 1;
109         memset(ct->buf, 0, blksize);
110         ct->io_starttime.tv_sec = 0;
111         ct->io_starttime.tv_nsec = 0;
112
113         return 0;
114 }
115
116 static void deinit_each_dio_ctx(struct dio_ctx *ct)
117 {
118         if (ct->buf)
119                 free(ct->buf);
120 }
121
122 static int setup_directio_ctx(struct io_err_stat_path *p)
123 {
124         unsigned long pgsize = getpagesize();
125         char fpath[PATH_MAX];
126         int blksize = 0;
127         int i;
128
129         if (snprintf(fpath, PATH_MAX, "/dev/%s", p->devname) >= PATH_MAX)
130                 return 1;
131         if (p->fd < 0)
132                 p->fd = open(fpath, O_RDONLY | O_DIRECT);
133         if (p->fd < 0)
134                 return 1;
135
136         p->dio_ctx_array = MALLOC(sizeof(struct dio_ctx) * CONCUR_NR_EVENT);
137         if (!p->dio_ctx_array)
138                 goto fail_close;
139
140         if (ioctl(p->fd, BLKBSZGET, &blksize) < 0) {
141                 io_err_stat_log(4, "%s:cannot get blocksize, set default 512",
142                                 p->devname);
143                 blksize = 512;
144         }
145         if (!blksize)
146                 goto free_pdctx;
147
148         for (i = 0; i < CONCUR_NR_EVENT; i++) {
149                 if (init_each_dio_ctx(p->dio_ctx_array + i, blksize, pgsize))
150                         goto deinit;
151         }
152         return 0;
153
154 deinit:
155         for (i = 0; i < CONCUR_NR_EVENT; i++)
156                 deinit_each_dio_ctx(p->dio_ctx_array + i);
157 free_pdctx:
158         FREE(p->dio_ctx_array);
159 fail_close:
160         close(p->fd);
161
162         return 1;
163 }
164
165 static void destroy_directio_ctx(struct io_err_stat_path *p)
166 {
167         int i;
168
169         if (!p || !p->dio_ctx_array)
170                 return;
171         cancel_inflight_io(p);
172
173         for (i = 0; i < CONCUR_NR_EVENT; i++)
174                 deinit_each_dio_ctx(p->dio_ctx_array + i);
175         FREE(p->dio_ctx_array);
176
177         if (p->fd > 0)
178                 close(p->fd);
179 }
180
181 static struct io_err_stat_path *alloc_io_err_stat_path(void)
182 {
183         struct io_err_stat_path *p;
184
185         p = (struct io_err_stat_path *)MALLOC(sizeof(*p));
186         if (!p)
187                 return NULL;
188
189         memset(p->devname, 0, sizeof(p->devname));
190         p->io_err_nr = 0;
191         p->io_nr = 0;
192         p->total_time = 0;
193         p->start_time.tv_sec = 0;
194         p->start_time.tv_nsec = 0;
195         p->err_rate_threshold = 0;
196         p->fd = -1;
197
198         return p;
199 }
200
201 static void free_io_err_stat_path(struct io_err_stat_path *p)
202 {
203         FREE(p);
204 }
205
206 static struct io_err_stat_pathvec *alloc_pathvec(void)
207 {
208         struct io_err_stat_pathvec *p;
209         int r;
210
211         p = (struct io_err_stat_pathvec *)MALLOC(sizeof(*p));
212         if (!p)
213                 return NULL;
214         p->pathvec = vector_alloc();
215         if (!p->pathvec)
216                 goto out_free_struct_pathvec;
217         r = pthread_mutex_init(&p->mutex, NULL);
218         if (r)
219                 goto out_free_member_pathvec;
220
221         return p;
222
223 out_free_member_pathvec:
224         vector_free(p->pathvec);
225 out_free_struct_pathvec:
226         FREE(p);
227         return NULL;
228 }
229
230 static void free_io_err_pathvec(struct io_err_stat_pathvec *p)
231 {
232         struct io_err_stat_path *path;
233         int i;
234
235         if (!p)
236                 return;
237         pthread_mutex_destroy(&p->mutex);
238         if (!p->pathvec) {
239                 vector_foreach_slot(p->pathvec, path, i) {
240                         destroy_directio_ctx(path);
241                         free_io_err_stat_path(path);
242                 }
243                 vector_free(p->pathvec);
244         }
245         FREE(p);
246 }
247
248 /*
249  * return value
250  * 0: enqueue OK
251  * 1: fails because of internal error
252  * 2: fails because of existing already
253  */
254 static int enqueue_io_err_stat_by_path(struct path *path)
255 {
256         struct io_err_stat_path *p;
257
258         pthread_mutex_lock(&paths->mutex);
259         p = find_err_path_by_dev(paths->pathvec, path->dev);
260         if (p) {
261                 pthread_mutex_unlock(&paths->mutex);
262                 return 2;
263         }
264         pthread_mutex_unlock(&paths->mutex);
265
266         p = alloc_io_err_stat_path();
267         if (!p)
268                 return 1;
269
270         memcpy(p->devname, path->dev, sizeof(p->devname));
271         p->total_time = path->mpp->marginal_path_err_sample_time;
272         p->err_rate_threshold = path->mpp->marginal_path_err_rate_threshold;
273
274         if (setup_directio_ctx(p))
275                 goto free_ioerr_path;
276         pthread_mutex_lock(&paths->mutex);
277         if (!vector_alloc_slot(paths->pathvec))
278                 goto unlock_destroy;
279         vector_set_slot(paths->pathvec, p);
280         pthread_mutex_unlock(&paths->mutex);
281
282         if (!path->io_err_disable_reinstate) {
283                 /*
284                  *fail the path in the kernel for the time of the to make
285                  *the test more reliable
286                  */
287                 io_err_stat_log(3, "%s: fail dm path %s before checking",
288                                 path->mpp->alias, path->dev);
289                 path->io_err_disable_reinstate = 1;
290                 dm_fail_path(path->mpp->alias, path->dev_t);
291                 update_queue_mode_del_path(path->mpp);
292
293                 /*
294                  * schedule path check as soon as possible to
295                  * update path state to delayed state
296                  */
297                 path->tick = 1;
298
299         }
300         io_err_stat_log(2, "%s: enqueue path %s to check",
301                         path->mpp->alias, path->dev);
302         return 0;
303
304 unlock_destroy:
305         pthread_mutex_unlock(&paths->mutex);
306         destroy_directio_ctx(p);
307 free_ioerr_path:
308         free_io_err_stat_path(p);
309
310         return 1;
311 }
312
313 int io_err_stat_handle_pathfail(struct path *path)
314 {
315         struct timespec curr_time;
316         int res;
317
318         if (path->io_err_disable_reinstate) {
319                 io_err_stat_log(3, "%s: reinstate is already disabled",
320                                 path->dev);
321                 return 1;
322         }
323         if (path->io_err_pathfail_cnt < 0)
324                 return 1;
325
326         if (!path->mpp)
327                 return 1;
328         if (path->mpp->nr_active <= 1)
329                 return 1;
330         if (path->mpp->marginal_path_double_failed_time <= 0 ||
331                 path->mpp->marginal_path_err_sample_time <= 0 ||
332                 path->mpp->marginal_path_err_recheck_gap_time <= 0 ||
333                 path->mpp->marginal_path_err_rate_threshold < 0) {
334                 io_err_stat_log(4, "%s: parameter not set", path->mpp->alias);
335                 return 1;
336         }
337         if (path->mpp->marginal_path_err_sample_time < (2 * IOTIMEOUT_SEC)) {
338                 io_err_stat_log(2, "%s: marginal_path_err_sample_time should not less than %d",
339                                 path->mpp->alias, 2 * IOTIMEOUT_SEC);
340                 return 1;
341         }
342         /*
343          * The test should only be started for paths that have failed
344          * repeatedly in a certain time frame, so that we have reason
345          * to assume they're flaky. Without bother the admin to configure
346          * the repeated count threshold and time frame, we assume a path
347          * which fails at least twice within 60 seconds is flaky.
348          */
349         if (clock_gettime(CLOCK_MONOTONIC, &curr_time) != 0)
350                 return 1;
351         if (path->io_err_pathfail_cnt == 0) {
352                 path->io_err_pathfail_cnt++;
353                 path->io_err_pathfail_starttime = curr_time.tv_sec;
354                 io_err_stat_log(5, "%s: start path flakiness pre-checking",
355                                 path->dev);
356                 return 0;
357         }
358         if ((curr_time.tv_sec - path->io_err_pathfail_starttime) >
359                         path->mpp->marginal_path_double_failed_time) {
360                 path->io_err_pathfail_cnt = 0;
361                 path->io_err_pathfail_starttime = curr_time.tv_sec;
362                 io_err_stat_log(5, "%s: restart path flakiness pre-checking",
363                                 path->dev);
364         }
365         path->io_err_pathfail_cnt++;
366         if (path->io_err_pathfail_cnt >= FLAKY_PATHFAIL_THRESHOLD) {
367                 res = enqueue_io_err_stat_by_path(path);
368                 if (!res)
369                         path->io_err_pathfail_cnt = PATH_IO_ERR_IN_CHECKING;
370                 else
371                         path->io_err_pathfail_cnt = 0;
372         }
373
374         return 0;
375 }
376
377 int hit_io_err_recheck_time(struct path *pp)
378 {
379         struct timespec curr_time;
380         int r;
381
382         if (pp->io_err_disable_reinstate == 0)
383                 return 1;
384         if (clock_gettime(CLOCK_MONOTONIC, &curr_time) != 0)
385                 return 1;
386         if (pp->io_err_pathfail_cnt != PATH_IO_ERR_IN_POLLING_RECHECK)
387                 return 1;
388         if (pp->mpp->nr_active <= 0) {
389                 io_err_stat_log(2, "%s: recover path early", pp->dev);
390                 goto recover;
391         }
392         if ((curr_time.tv_sec - pp->io_err_dis_reinstate_time) >
393                         pp->mpp->marginal_path_err_recheck_gap_time) {
394                 io_err_stat_log(4, "%s: reschedule checking after %d seconds",
395                                 pp->dev,
396                                 pp->mpp->marginal_path_err_recheck_gap_time);
397                 /*
398                  * to reschedule io error checking again
399                  * if the path is good enough, we claim it is good
400                  * and can be reinsated as soon as possible in the
401                  * check_path routine.
402                  */
403                 pp->io_err_dis_reinstate_time = curr_time.tv_sec;
404                 r = enqueue_io_err_stat_by_path(pp);
405                 /*
406                  * Enqueue fails because of internal error.
407                  * In this case , we recover this path
408                  * Or else,  return 1 to set path state to PATH_SHAKY
409                  */
410                 if (r == 1) {
411                         io_err_stat_log(3, "%s: enqueue fails, to recover",
412                                         pp->dev);
413                         goto recover;
414                 } else if (!r) {
415                         pp->io_err_pathfail_cnt = PATH_IO_ERR_IN_CHECKING;
416                 }
417         }
418
419         return 1;
420
421 recover:
422         pp->io_err_pathfail_cnt = 0;
423         pp->io_err_disable_reinstate = 0;
424         pp->tick = 1;
425         return 0;
426 }
427
428 static int delete_io_err_stat_by_addr(struct io_err_stat_path *p)
429 {
430         int i;
431
432         i = find_slot(paths->pathvec, p);
433         if (i != -1)
434                 vector_del_slot(paths->pathvec, i);
435
436         destroy_directio_ctx(p);
437         free_io_err_stat_path(p);
438
439         return 0;
440 }
441
442 static void account_async_io_state(struct io_err_stat_path *pp, int rc)
443 {
444         switch (rc) {
445         case PATH_DOWN:
446         case PATH_TIMEOUT:
447                 pp->io_err_nr++;
448                 break;
449         case PATH_UNCHECKED:
450         case PATH_UP:
451         case PATH_PENDING:
452                 break;
453         default:
454                 break;
455         }
456 }
457
458 static int poll_io_err_stat(struct vectors *vecs, struct io_err_stat_path *pp)
459 {
460         struct timespec currtime, difftime;
461         struct path *path;
462         double err_rate;
463
464         if (clock_gettime(CLOCK_MONOTONIC, &currtime) != 0)
465                 return 1;
466         timespecsub(&currtime, &pp->start_time, &difftime);
467         if (difftime.tv_sec < pp->total_time)
468                 return 0;
469
470         io_err_stat_log(4, "%s: check end", pp->devname);
471
472         err_rate = pp->io_nr == 0 ? 0 : (pp->io_err_nr * 1000.0f) / pp->io_nr;
473         io_err_stat_log(3, "%s: IO error rate (%.1f/1000)",
474                         pp->devname, err_rate);
475         pthread_cleanup_push(cleanup_lock, &vecs->lock);
476         lock(&vecs->lock);
477         pthread_testcancel();
478         path = find_path_by_dev(vecs->pathvec, pp->devname);
479         if (!path) {
480                 io_err_stat_log(4, "path %s not found'", pp->devname);
481         } else if (err_rate <= pp->err_rate_threshold) {
482                 path->io_err_pathfail_cnt = 0;
483                 path->io_err_disable_reinstate = 0;
484                 io_err_stat_log(3, "%s: (%d/%d) good to enable reinstating",
485                                 pp->devname, pp->io_err_nr, pp->io_nr);
486                 /*
487                  * schedule path check as soon as possible to
488                  * update path state. Do NOT reinstate dm path here
489                  */
490                 path->tick = 1;
491
492         } else if (path->mpp && path->mpp->nr_active > 1) {
493                 io_err_stat_log(3, "%s: keep failing the dm path %s",
494                                 path->mpp->alias, path->dev);
495                 path->io_err_pathfail_cnt = PATH_IO_ERR_IN_POLLING_RECHECK;
496                 path->io_err_disable_reinstate = 1;
497                 path->io_err_dis_reinstate_time = currtime.tv_sec;
498                 io_err_stat_log(3, "%s: disable reinstating of %s",
499                                 path->mpp->alias, path->dev);
500         } else {
501                 path->io_err_pathfail_cnt = 0;
502                 path->io_err_disable_reinstate = 0;
503                 io_err_stat_log(3, "%s: there is orphan path, enable reinstating",
504                                 pp->devname);
505         }
506         lock_cleanup_pop(vecs->lock);
507
508         delete_io_err_stat_by_addr(pp);
509
510         return 0;
511 }
512
513 static int send_each_async_io(struct dio_ctx *ct, int fd, char *dev)
514 {
515         int rc = -1;
516
517         if (ct->io_starttime.tv_nsec == 0 &&
518                         ct->io_starttime.tv_sec == 0) {
519                 struct iocb *ios[1] = { &ct->io };
520
521                 if (clock_gettime(CLOCK_MONOTONIC, &ct->io_starttime) != 0) {
522                         ct->io_starttime.tv_sec = 0;
523                         ct->io_starttime.tv_nsec = 0;
524                         return rc;
525                 }
526                 io_prep_pread(&ct->io, fd, ct->buf, ct->blksize, 0);
527                 if (io_submit(ioctx, 1, ios) != 1) {
528                         io_err_stat_log(5, "%s: io_submit error %i",
529                                         dev, errno);
530                         return rc;
531                 }
532                 rc = 0;
533         }
534
535         return rc;
536 }
537
538 static void send_batch_async_ios(struct io_err_stat_path *pp)
539 {
540         int i;
541         struct dio_ctx *ct;
542         struct timespec currtime, difftime;
543
544         if (clock_gettime(CLOCK_MONOTONIC, &currtime) != 0)
545                 return;
546         /*
547          * Give a free time for all IO to complete or timeout
548          */
549         if (pp->start_time.tv_sec != 0) {
550                 timespecsub(&currtime, &pp->start_time, &difftime);
551                 if (difftime.tv_sec + IOTIMEOUT_SEC >= pp->total_time)
552                         return;
553         }
554
555         for (i = 0; i < CONCUR_NR_EVENT; i++) {
556                 ct = pp->dio_ctx_array + i;
557                 if (!send_each_async_io(ct, pp->fd, pp->devname))
558                         pp->io_nr++;
559         }
560         if (pp->start_time.tv_sec == 0 && pp->start_time.tv_nsec == 0 &&
561                 clock_gettime(CLOCK_MONOTONIC, &pp->start_time)) {
562                 pp->start_time.tv_sec = 0;
563                 pp->start_time.tv_nsec = 0;
564         }
565 }
566
567 static int try_to_cancel_timeout_io(struct dio_ctx *ct, struct timespec *t,
568                 char *dev)
569 {
570         struct timespec difftime;
571         struct io_event event;
572         int             rc = PATH_UNCHECKED;
573         int             r;
574
575         if (ct->io_starttime.tv_sec == 0)
576                 return rc;
577         timespecsub(t, &ct->io_starttime, &difftime);
578         if (difftime.tv_sec > IOTIMEOUT_SEC) {
579                 struct iocb *ios[1] = { &ct->io };
580
581                 io_err_stat_log(5, "%s: abort check on timeout", dev);
582                 r = io_cancel(ioctx, ios[0], &event);
583                 if (r)
584                         io_err_stat_log(5, "%s: io_cancel error %i",
585                                         dev, errno);
586                 ct->io_starttime.tv_sec = 0;
587                 ct->io_starttime.tv_nsec = 0;
588                 rc = PATH_TIMEOUT;
589         } else {
590                 rc = PATH_PENDING;
591         }
592
593         return rc;
594 }
595
596 static void poll_async_io_timeout(void)
597 {
598         struct io_err_stat_path *pp;
599         struct timespec curr_time;
600         int             rc = PATH_UNCHECKED;
601         int             i, j;
602
603         if (clock_gettime(CLOCK_MONOTONIC, &curr_time) != 0)
604                 return;
605         vector_foreach_slot(paths->pathvec, pp, i) {
606                 for (j = 0; j < CONCUR_NR_EVENT; j++) {
607                         rc = try_to_cancel_timeout_io(pp->dio_ctx_array + j,
608                                         &curr_time, pp->devname);
609                         account_async_io_state(pp, rc);
610                 }
611         }
612 }
613
614 static void cancel_inflight_io(struct io_err_stat_path *pp)
615 {
616         struct io_event event;
617         int i, r;
618
619         for (i = 0; i < CONCUR_NR_EVENT; i++) {
620                 struct dio_ctx *ct = pp->dio_ctx_array + i;
621                 struct iocb *ios[1] = { &ct->io };
622
623                 if (ct->io_starttime.tv_sec == 0
624                                 && ct->io_starttime.tv_nsec == 0)
625                         continue;
626                 io_err_stat_log(5, "%s: abort infligh io",
627                                 pp->devname);
628                 r = io_cancel(ioctx, ios[0], &event);
629                 if (r)
630                         io_err_stat_log(5, "%s: io_cancel error %d, %i",
631                                         pp->devname, r, errno);
632                 ct->io_starttime.tv_sec = 0;
633                 ct->io_starttime.tv_nsec = 0;
634         }
635 }
636
637 static inline int handle_done_dio_ctx(struct dio_ctx *ct, struct io_event *ev)
638 {
639         ct->io_starttime.tv_sec = 0;
640         ct->io_starttime.tv_nsec = 0;
641         return (ev->res == ct->blksize) ? PATH_UP : PATH_DOWN;
642 }
643
644 static void handle_async_io_done_event(struct io_event *io_evt)
645 {
646         struct io_err_stat_path *pp;
647         struct dio_ctx *ct;
648         int rc = PATH_UNCHECKED;
649         int i, j;
650
651         vector_foreach_slot(paths->pathvec, pp, i) {
652                 for (j = 0; j < CONCUR_NR_EVENT; j++) {
653                         ct = pp->dio_ctx_array + j;
654                         if (&ct->io == io_evt->obj) {
655                                 rc = handle_done_dio_ctx(ct, io_evt);
656                                 account_async_io_state(pp, rc);
657                                 return;
658                         }
659                 }
660         }
661 }
662
663 static void process_async_ios_event(int timeout_nsecs, char *dev)
664 {
665         struct io_event events[CONCUR_NR_EVENT];
666         int             i, n;
667         struct timespec timeout = { .tv_nsec = timeout_nsecs };
668
669         errno = 0;
670         n = io_getevents(ioctx, 1L, CONCUR_NR_EVENT, events, &timeout);
671         if (n < 0) {
672                 io_err_stat_log(3, "%s: async io events returned %d (errno=%s)",
673                                 dev, n, strerror(errno));
674         } else {
675                 for (i = 0; i < n; i++)
676                         handle_async_io_done_event(&events[i]);
677         }
678 }
679
680 static void service_paths(void)
681 {
682         struct io_err_stat_path *pp;
683         int i;
684
685         pthread_mutex_lock(&paths->mutex);
686         vector_foreach_slot(paths->pathvec, pp, i) {
687                 send_batch_async_ios(pp);
688                 process_async_ios_event(TIMEOUT_NO_IO_NSEC, pp->devname);
689                 poll_async_io_timeout();
690                 poll_io_err_stat(vecs, pp);
691         }
692         pthread_mutex_unlock(&paths->mutex);
693 }
694
695 static void *io_err_stat_loop(void *data)
696 {
697         vecs = (struct vectors *)data;
698         pthread_cleanup_push(rcu_unregister, NULL);
699         rcu_register_thread();
700
701         mlockall(MCL_CURRENT | MCL_FUTURE);
702         while (1) {
703                 service_paths();
704                 usleep(100000);
705         }
706
707         pthread_cleanup_pop(1);
708         return NULL;
709 }
710
711 int start_io_err_stat_thread(void *data)
712 {
713         if (io_setup(CONCUR_NR_EVENT, &ioctx) != 0) {
714                 io_err_stat_log(4, "io_setup failed");
715                 return 1;
716         }
717         paths = alloc_pathvec();
718         if (!paths)
719                 goto destroy_ctx;
720
721         if (pthread_create(&io_err_stat_thr, &io_err_stat_attr,
722                                 io_err_stat_loop, data)) {
723                 io_err_stat_log(0, "cannot create io_error statistic thread");
724                 goto out_free;
725         }
726         io_err_stat_log(3, "thread started");
727         return 0;
728
729 out_free:
730         free_io_err_pathvec(paths);
731 destroy_ctx:
732         io_destroy(ioctx);
733         io_err_stat_log(0, "failed to start io_error statistic thread");
734         return 1;
735 }
736
737 void stop_io_err_stat_thread(void)
738 {
739         pthread_cancel(io_err_stat_thr);
740         pthread_kill(io_err_stat_thr, SIGUSR2);
741         free_io_err_pathvec(paths);
742         io_destroy(ioctx);
743 }