From b0b37bcd657142d942e7ee8b1c7f1d3651db2d29 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Mon, 23 Jun 2008 14:00:57 +0200 Subject: [PATCH] [MEDIUM] further improve monotonic clock by check forward jumps The first implementation of the monotonic clock did not verify forward jumps. The consequence is that a fast changing time may expire a lot of tasks. While it does seem minor, in fact it is problematic because most machines which boot with a wrong date are in the past and suddenly see their time jump by several years in the future. The solution is to check if we spent more apparent time in a poller than allowed (with a margin applied). The margin is currently set to 1000 ms. It should be large enough for any poll() to complete. Tests with randomly jumping clock show that the result is quite accurate (error less than 1 second at every change of more than one second). --- include/common/time.h | 19 +++++++++----- src/cfgparse.c | 2 +- src/ev_epoll.c | 9 ++++--- src/ev_kqueue.c | 34 +++++++++++++++++------- src/ev_poll.c | 9 ++++--- src/ev_select.c | 24 ++++++++++++++--- src/ev_sepoll.c | 12 +++++---- src/haproxy.c | 4 +-- src/proxy.c | 6 ++--- src/time.c | 61 ++++++++++++++++++++++++++++++++----------- 10 files changed, 129 insertions(+), 51 deletions(-) diff --git a/include/common/time.h b/include/common/time.h index d6155265d..82ec402cc 100644 --- a/include/common/time.h +++ b/include/common/time.h @@ -44,6 +44,11 @@ #define TIME_ETERNITY (TV_ETERNITY_MS) +/* we want to be able to detect time jumps. Fix the maximum wait time to a low + * value so that we know the time has changed if we wait longer. + */ +#define MAX_DELAY_MS 1000 + /* returns the lowest delay amongst and , and respects TIME_ETERNITY */ #define MINTIME(old, new) (((new)<0)?(old):(((old)<0||(new)<(old))?(new):(old))) @@ -84,13 +89,15 @@ REGPRM1 static inline struct timeval *tv_now(struct timeval *tv) return tv; } -/* tv_now_mono: sets to the current time (wall clock), to a value - * following a monotonic function, and applies any required correction if the - * time goes backwards. Note that while we could improve it a bit by checking - * that the new date is not too far in the future, it is not much necessary to - * do so. +/* tv_udpate_date: sets to system time, and sets to something as + * close as possible to real time, following a monotonic function. The main + * principle consists in detecting backwards and forwards time jumps and adjust + * an offset to correct them. This function should be called only once after + * each poll. The poll's timeout should be passed in , and the return + * value in (a non-zero value means that we have not expired the + * timeout). */ -REGPRM2 struct timeval *tv_now_mono(struct timeval *mono, struct timeval *wall); +REGPRM2 void tv_update_date(int max_wait, int interrupted); /* * sets a struct timeval to its highest value so that it can never happen diff --git a/src/cfgparse.c b/src/cfgparse.c index 13d2a8930..90611cdaa 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -2831,7 +2831,7 @@ int readcfgfile(const char *file) */ /* will be needed further to delay some tasks */ - tv_now_mono(&now, &date); + tv_update_date(0,1); if ((curproxy = proxy) == NULL) { Alert("parsing %s : no line. Nothing to do !\n", diff --git a/src/ev_epoll.c b/src/ev_epoll.c index e7aea93f3..adb0fd6e8 100644 --- a/src/ev_epoll.c +++ b/src/ev_epoll.c @@ -235,15 +235,18 @@ REGPRM2 static void _do_poll(struct poller *p, struct timeval *exp) if (run_queue) wait_time = 0; else if (tv_iseternity(exp)) - wait_time = -1; + wait_time = MAX_DELAY_MS; else if (tv_isge(&now, exp)) wait_time = 0; - else + else { wait_time = __tv_ms_elapsed(&now, exp) + 1; + if (wait_time > MAX_DELAY_MS) + wait_time = MAX_DELAY_MS; + } fd = MIN(maxfd, global.tune.maxpollevents); status = epoll_wait(epoll_fd, epoll_events, fd, wait_time); - tv_now_mono(&now, &date); + tv_update_date(wait_time, status); for (count = 0; count < status; count++) { fd = epoll_events[count].data.fd; diff --git a/src/ev_kqueue.c b/src/ev_kqueue.c index f22aa5b1f..71e9ecf61 100644 --- a/src/ev_kqueue.c +++ b/src/ev_kqueue.c @@ -102,25 +102,41 @@ REGPRM1 static void __fd_clo(int fd) REGPRM2 static void _do_poll(struct poller *p, struct timeval *exp) { int status; - int count, fd; - struct timespec timeout, *to_ptr; + int count, fd, delta_ms; + struct timespec timeout; - to_ptr = NULL; // no timeout if (run_queue) { timeout.tv_sec = timeout.tv_nsec = 0; - to_ptr = &timeout; + delta_ms = 0; } else if (tv_isset(exp)) { + const struct timeval max_delay = { + .tv_sec = MAX_DELAY_MS / 1000, + .tv_usec = (MAX_DELAY_MS % 1000) * 1000 + }; struct timeval delta; - if (tv_isge(&now, exp)) + if (tv_isge(&now, exp)) { delta.tv_sec = delta.tv_usec = 0; - else + delta_ms = 0; + } + else { tv_remain(&now, exp, &delta); + if (__tv_isgt(&delta, &max_delay)) { + delta = max_delay; + delta_ms = MAX_DELAY_MS; + } else { + delta_ms = delta.tv_sec * 1000 + delta.tv_usec / 1000; + } + } timeout.tv_sec = delta.tv_sec; timeout.tv_nsec = delta.tv_usec * 1000; - to_ptr = &timeout; + } + else { + delta_ms = MAX_DELAY_MS; + timeout.tv_sec = MAX_DELAY_MS / 1000; + timeout.tv_nsec = (MAX_DELAY_MS % 1000) * 1000000; } fd = MIN(maxfd, global.tune.maxpollevents); @@ -129,8 +145,8 @@ REGPRM2 static void _do_poll(struct poller *p, struct timeval *exp) 0, // int nchanges kev, // struct kevent *eventlist fd, // int nevents - to_ptr); // const struct timespec *timeout - tv_now_mono(&now, &date); + &timeout); // const struct timespec *timeout + tv_update_date(delta_ms, status); for (count = 0; count < status; count++) { fd = kev[count].ident; diff --git a/src/ev_poll.c b/src/ev_poll.c index bfbe999ed..a0355aad6 100644 --- a/src/ev_poll.c +++ b/src/ev_poll.c @@ -127,14 +127,17 @@ REGPRM2 static void _do_poll(struct poller *p, struct timeval *exp) if (run_queue) wait_time = 0; else if (tv_iseternity(exp)) - wait_time = -1; + wait_time = MAX_DELAY_MS; else if (tv_isge(&now, exp)) wait_time = 0; - else + else { wait_time = __tv_ms_elapsed(&now, exp) + 1; + if (wait_time > MAX_DELAY_MS) + wait_time = MAX_DELAY_MS; + } status = poll(poll_events, nbfd, wait_time); - tv_now_mono(&now, &date); + tv_update_date(wait_time, status); for (count = 0; status > 0 && count < nbfd; count++) { fd = poll_events[count].fd; diff --git a/src/ev_select.c b/src/ev_select.c index 25bd3ec60..30df9285b 100644 --- a/src/ev_select.c +++ b/src/ev_select.c @@ -80,17 +80,27 @@ REGPRM1 static void __fd_rem(int fd) */ REGPRM2 static void _do_poll(struct poller *p, struct timeval *exp) { + const struct timeval max_delay = { + .tv_sec = MAX_DELAY_MS / 1000, + .tv_usec = (MAX_DELAY_MS % 1000) * 1000 + }; int status; int fd, i; struct timeval delta; + int delta_ms; int readnotnull, writenotnull; int fds; char count; /* allow select to return immediately when needed */ delta.tv_sec = delta.tv_usec = 0; - if (!run_queue && tv_isset(exp)) { - if (tv_islt(&now, exp)) { + delta_ms = 0; + if (!run_queue) { + if (!tv_isset(exp)) { + delta = max_delay; + delta_ms = MAX_DELAY_MS; + } + else if (tv_islt(&now, exp)) { tv_remain(&now, exp, &delta); /* To avoid eventual select loops due to timer precision */ delta.tv_usec += SCHEDULER_RESOLUTION * 1000; @@ -98,6 +108,12 @@ REGPRM2 static void _do_poll(struct poller *p, struct timeval *exp) delta.tv_usec -= 1000000; delta.tv_sec ++; } + if (__tv_isge(&delta, &max_delay)) { + delta = max_delay; + delta_ms = MAX_DELAY_MS; + } else { + delta_ms = delta.tv_sec * 1000 + delta.tv_usec / 1000; + } } } @@ -122,9 +138,9 @@ REGPRM2 static void _do_poll(struct poller *p, struct timeval *exp) readnotnull ? tmp_evts[DIR_RD] : NULL, writenotnull ? tmp_evts[DIR_WR] : NULL, NULL, - tv_isset(exp) ? &delta : NULL); + &delta); - tv_now_mono(&now, &date); + tv_update_date(delta_ms, status); if (status <= 0) return; diff --git a/src/ev_sepoll.c b/src/ev_sepoll.c index ed2103ced..f42a97f29 100644 --- a/src/ev_sepoll.c +++ b/src/ev_sepoll.c @@ -418,7 +418,7 @@ REGPRM2 static void _do_poll(struct poller *p, struct timeval *exp) * returning now without checking epoll_wait(). */ if (++last_skipped <= 1) { - tv_now_mono(&now, &date); + tv_update_date(0, 1); return; } } @@ -435,11 +435,14 @@ REGPRM2 static void _do_poll(struct poller *p, struct timeval *exp) } else { if (tv_iseternity(exp)) - wait_time = -1; + wait_time = MAX_DELAY_MS; else if (tv_isge(&now, exp)) wait_time = 0; - else + else { wait_time = __tv_ms_elapsed(&now, exp) + 1; + if (wait_time > MAX_DELAY_MS) + wait_time = MAX_DELAY_MS; + } } /* now let's wait for real events. We normally use maxpollevents as a @@ -451,8 +454,7 @@ REGPRM2 static void _do_poll(struct poller *p, struct timeval *exp) fd = MIN(maxfd, fd); spec_processed = 0; status = epoll_wait(epoll_fd, epoll_events, fd, wait_time); - - tv_now_mono(&now, &date); + tv_update_date(wait_time, status); for (count = 0; count < status; count++) { int e = epoll_events[count].events; diff --git a/src/haproxy.c b/src/haproxy.c index f10e47dc4..45e48522d 100644 --- a/src/haproxy.c +++ b/src/haproxy.c @@ -415,7 +415,7 @@ void init(int argc, char **argv) global.rlimit_memmax = HAPROXY_MEMMAX; #endif - tv_now_mono(&now, &date); + tv_update_date(-1,-1); start_date = now; init_task(); @@ -897,7 +897,7 @@ void run_poll_loop() { struct timeval next; - tv_now_mono(&now, &date); + tv_update_date(0,1); while (1) { process_runnable_tasks(&next); diff --git a/src/proxy.c b/src/proxy.c index 16804f913..a7b4efc1d 100644 --- a/src/proxy.c +++ b/src/proxy.c @@ -385,7 +385,7 @@ void soft_stop(void) stopping = 1; p = proxy; - tv_now_mono(&now, &date); /* else, the old time before select will be used */ + tv_update_date(0,1); /* else, the old time before select will be used */ while (p) { if (p->state != PR_STSTOPPED) { Warning("Stopping proxy %s in %d ms.\n", p->id, p->grace); @@ -434,7 +434,7 @@ void pause_proxies(void) err = 0; p = proxy; - tv_now_mono(&now, &date); /* else, the old time before select will be used */ + tv_update_date(0,1); /* else, the old time before select will be used */ while (p) { if (p->state != PR_STERROR && p->state != PR_STSTOPPED && @@ -469,7 +469,7 @@ void listen_proxies(void) struct listener *l; p = proxy; - tv_now_mono(&now, &date); /* else, the old time before select will be used */ + tv_update_date(0,1); /* else, the old time before select will be used */ while (p) { if (p->state == PR_STPAUSED) { Warning("Enabling proxy %s.\n", p->id); diff --git a/src/time.c b/src/time.c index ccb30b253..f637f6c78 100644 --- a/src/time.c +++ b/src/time.c @@ -143,25 +143,56 @@ REGPRM2 int _tv_isgt(const struct timeval *tv1, const struct timeval *tv2) return __tv_isgt(tv1, tv2); } -/* tv_now_mono: sets to the current time (wall clock), to a value - * following a monotonic function, and applies any required correction if the - * time goes backwards. Note that while we could improve it a bit by checking - * that the new date is not too far in the future, it is not much necessary to - * do so. +/* tv_udpate_date: sets to system time, and sets to something as + * close as possible to real time, following a monotonic function. The main + * principle consists in detecting backwards and forwards time jumps and adjust + * an offset to correct them. This function should be called once after each + * poll, and never farther apart than MAX_DELAY_MS*2. The poll's timeout should + * be passed in , and the return value in (a non-zero + * value means that we have not expired the timeout). Calling it with (-1,*) + * sets both and to current date, and calling it with (0,1) simply + * updates the values. */ -REGPRM2 struct timeval *tv_now_mono(struct timeval *mono, struct timeval *wall) +REGPRM2 void tv_update_date(int max_wait, int interrupted) { - static struct timeval tv_offset; - struct timeval adjusted; + static struct timeval tv_offset; /* warning: signed offset! */ + struct timeval adjusted, deadline; - gettimeofday(wall, NULL); - __tv_add(&adjusted, wall, &tv_offset); - if (unlikely(__tv_islt(&adjusted, mono))) { - __tv_remain(wall, mono, &tv_offset); - return mono; + gettimeofday(&date, NULL); + if (unlikely(max_wait < 0)) { + tv_zero(&tv_offset); + now = date; + return; } - *mono = adjusted; - return mono; + __tv_add(&adjusted, &date, &tv_offset); + if (unlikely(__tv_islt(&adjusted, &now))) { + goto fixup; /* jump in the past */ + } + + /* OK we did not jump backwards, let's see if we have jumped too far + * forwards. The poll value was in , we accept that plus + * MAX_DELAY_MS to cover additional time. + */ + _tv_ms_add(&deadline, &now, max_wait + MAX_DELAY_MS); + if (unlikely(__tv_isge(&adjusted, &deadline))) { + goto fixup; /* jump in the future */ + } + now = adjusted; + return; + fixup: + /* Large jump. If the poll was interrupted, we consider that the date + * has not changed (immediate wake-up), otherwise we add the poll + * time-out to the previous date. The new offset is recomputed. + */ + if (!interrupted) + _tv_ms_add(&now, &now, max_wait); + tv_offset.tv_sec = now.tv_sec - date.tv_sec; + tv_offset.tv_usec = now.tv_usec - date.tv_usec; + if (tv_offset.tv_usec < 0) { + tv_offset.tv_usec += 1000000; + tv_offset.tv_sec--; + } + return; } char *human_time(int t, short hz_div) {