function deCalendarize(duration) { return Duration.new(Duration.seconds(duration)); } sub put_trend(field, over, every=null) { // estimate trailing trend as the median duration-over-duration change of all samples // in a window of -over duration and/or point-to-point change (a variant of the Theil-Sen // estimator). This can use up to [2 * over] of historic data per point, but can begin producing // (noisy) point-to-point estimates after two points. // // trend consumes its input stream, and outputs points every -every with T as the estimated // change over -over, and the trend portion of field as field_trend. [field - field_trend] // is the de-trended series. Additionally, t0,field_0 are the time and value of the initial // point of the result batch, such that field_trend = field_0 + [time - t0] * T. (this allows // the estimated T to be joined with and de-trend a denser version of the input stream). // // reject the trend as being 0 if the quartile range Q1...Q3 contains 0 (be pessimistic about // trends, as we do not expect them in short-horizon operations data; but they will confound // seasonality if not accounted for). // put __over = deCalendarize(over), __every = deCalendarize(every ?? __over / 30) | put __bucket = Math.floor((time - Date.quantize(time, __over)) / __every) | put __change_over = delta(field,null) == null ? null : delta(field) * (over / delta(time,:forever:)) by __bucket // change since over ago, if we have historic data | put __change = __change_over ?? delta(field, 0) * (__over / delta(time,:forever:)) // sample-to-sample, for startup | put -over 2 * over __change = (count() <= 3 || (last(time) - first(time) < 2 * __every)) ? 0 : __change, // moar data, please!! __Q = percentile(__change,[0, 0.25, 0.5, 0.75]), __t0=first(time), __y0=first(field) ,__count = count() | put __T = (__Q[1] < 0 && __Q[3] > 0) ? 0 : __Q[2] // ignore trends around 0, else median | put *(field + "_T") = __y0 + __T * (time - __t0) / __over // trend portion of field's value //| remove __Q, __t0, __y0, __bucket }