Scott Anderson
Senior Technical Writer & Technical Lead of
the Docs team @ InfluxData
InfluxDB Tasks
Beyond Downsampling

© 2020 InfluxData. All rights reserved. 2
What is an InfluxDB task?
A scheduled Flux script

© 2020 InfluxData. All rights reserved. 3
Why tasks?
● Automate repeated operations
● A replacement for Continuous Queries
(CQs) in InfluxDB Cloud and InfluxDB
2.0

© 2020 InfluxData. All rights reserved. 4
Downsampling cascade

© 2020 InfluxData. All rights reserved. 5
CREATE CONTINUOUS QUERY "downsample-daily" ON "my-db"
BEGIN
SELECT mean("example-field")
INTO "average-example-measurement"
FROM "example-measurement"
GROUP BY time(1h)
END

© 2020 InfluxData. All rights reserved. 6
from(bucket: "my-bucket")
|> range(start: tasks.lastSuccess(orTime: -task.every)
|> filter(fn: (r) =>
r._measurement == "example-m" and
r._field == "value"
)
|> aggregateWindow(every: 1h, fn: mean())
|> to(bucket: "downsampled-daily", org: "my-org")

© 2020 InfluxData. All rights reserved. 7
Downsampling InfluxDB Template
github.com/influxdata/community-templates

© 2020 InfluxData. All rights reserved. 8

© 2020 InfluxData. All rights reserved. 9

and the potential of Flux
BEYOND Downsampling

Monitoring & Alerting

© 2020 InfluxData. All rights reserved. 12

© 2020 InfluxData. All rights reserved. 13
import "influxdata/influxdb/monitor"
import "influxdata/influxdb/tasks"
import "influxdata/influxdb/v1"
check = {_check_id: "00000000deadbeef", _check_name: "Average memory monitor", _type: "threshold", tags: {}}
task_data = from(bucket: "my-bucket")
|> range(start: tasks.lastSuccess(orTime: -task.every), stop: now())
|> filter(fn: (r) => r._measurement == "telegraf" and r._field == "mem")
|> mean()
ok = (r) => r._value <= 60.0
crit = (r) => r._value > 95.0
warn = (r) => r._value > 80.0 and r._value <= 95.0
info = (r) => r._value > 60.0 and r._value <= 80.0
messageFn = (r) => ("Average mem check: ${r.host}'s average memory usage is ${r._value}%.")
task_data
|> v1.fieldsAsCols()
|> monitor.check(data: check, messageFn: messageFn, ok: ok, crit: crit, warn: warn, info: info)
System-generated check task

© 2020 InfluxData. All rights reserved. 14
import "influxdata/influxdb/monitor"
import "influxdata/influxdb/tasks"
import "influxdata/influxdb/v1"
check = {_check_id: "00000000deadbeef", _check_name: "Average memory monitor", _type: "threshold", tags: {}}
task_data = from(bucket: "my-bucket")
|> range(start: tasks.lastSuccess(orTime: -task.every), stop: now())
|> filter(fn: (r) => r._measurement == "telegraf" and r._field == "mem")
|> mean()
ok = (r) => r._value <= 60.0
crit = (r) => r._value > 95.0
warn = (r) => r._value > 80.0 and r._value <= 95.0
info = (r) => r._value > 60.0 and r._value <= 80.0
messageFn = (r) => ("Average mem check: ${r.host}'s average memory usage is ${r._value}%.")
task_data
|> v1.fieldsAsCols()
|> monitor.check(data: check, messageFn: messageFn, ok: ok, crit: crit, warn: warn, info: info)
System-generated check task

© 2020 InfluxData. All rights reserved. 15
import "influxdata/influxdb/monitor"
import "influxdata/influxdb/tasks"
import "influxdata/influxdb/v1"
check = {_check_id: "00000000deadbeef", _check_name: "Average memory monitor", _type: "threshold", tags: {}}
task_data = from(bucket: "my-bucket")
|> range(start: tasks.lastSuccess(orTime: -task.every), stop: now())
|> filter(fn: (r) => r._measurement == "telegraf" and r._field == "mem")
|> mean()
ok = (r) => r._value <= 60.0
crit = (r) => r._value > 95.0
warn = (r) => r._value > 80.0 and r._value <= 95.0
info = (r) => r._value > 60.0 and r._value <= 80.0
messageFn = (r) => ("Average mem check: ${r.host}'s average memory usage is ${r._value}%.")
task_data
|> v1.fieldsAsCols()
|> monitor.check(data: check, messageFn: messageFn, ok: ok, crit: crit, warn: warn, info: info)
System-generated check task

© 2020 InfluxData. All rights reserved. 16

© 2020 InfluxData. All rights reserved. 17
import "influxdata/influxdb/monitor"
import "influxdata/influxdb/tasks"
import "influxdata/influxdb/v1"
check = {_check_id: "00000000deadbeef", _check_name: "Average memory monitor", _type: "threshold", tags: {}}
task_data = from(bucket: "my-bucket")
|> range(start: tasks.lastSuccess(orTime: -task.every), stop: now())
|> filter(fn: (r) => r._measurement == "telegraf" and r._field == "mem")
|> mean()
ok = (r) => r._value <= 60.0
crit = (r) => r._value > 95.0
warn = (r) => r._value > 80.0 and r._value <= 95.0
info = (r) => r._value > 60.0 and r._value <= 80.0
messageFn = (r) => ("Average mem check: ${r.host}'s average memory usage is ${r._value}%.")
task_data
|> v1.fieldsAsCols()
|> monitor.check(data: check, messageFn: messageFn, ok: ok, crit: crit, warn: warn, info: info)
System-generated check task

© 2020 InfluxData. All rights reserved. 18
import "influxdata/influxdb/monitor"
import "influxdata/influxdb/tasks"
import "influxdata/influxdb/v1"
check = {_check_id: "00000000deadbeef", _check_name: "Average memory monitor", _type: "threshold", tags: {}}
task_data = from(bucket: "my-bucket")
|> range(start: tasks.lastSuccess(orTime: -task.every), stop: now())
|> filter(fn: (r) => r._measurement == "telegraf" and r._field == "mem")
|> mean()
ok = (r) => r._value <= 60.0
crit = (r) => r._value > 95.0
warn = (r) => r._value > 80.0 and r._value <= 95.0
info = (r) => r._value > 60.0 and r._value <= 80.0
messageFn = (r) => ("Average mem check: ${r.host}'s average memory usage is ${r._value}%.")
task_data
|> v1.fieldsAsCols()
|> monitor.check(data: check, messageFn: messageFn, ok: ok, crit: crit, warn: warn, info: info)
System-generated check task

© 2020 InfluxData. All rights reserved. 19

© 2020 InfluxData. All rights reserved. 20
import "influxdata/influxdb/monitor"
import "influxdata/influxdb/slack"
endpoint = slack.endpoint(
url: "https://slack.com/api/chat.postMessage",
token: "SLACK_TOKEN")
)
monitor.from(
start: -task.every,
fn: (r) => r._level == "crit"
)
|> endpoint(mapFn: (r) => ({ r with channel: "alerts", text: r.message, color: "danger"}))()
System-generated notification task

© 2020 InfluxData. All rights reserved. 21
import "influxdata/influxdb/monitor"
import "influxdata/influxdb/slack"
endpoint = slack.endpoint(
url: "https://slack.com/api/chat.postMessage",
token: "SLACK_TOKEN")
)
monitor.from(
start: -task.every,
fn: (r) => r._level == "crit"
)
|> endpoint(mapFn: (r) => ({ r with channel: "alerts", text: r.message, color: "danger"}))()
System-generated notification task

© 2020 InfluxData. All rights reserved. 22
import "influxdata/influxdb/monitor"
import "influxdata/influxdb/slack"
endpoint = slack.endpoint(
url: "https://slack.com/api/chat.postMessage",
token: "SLACK_TOKEN")
)
monitor.from(
start: -task.every,
fn: (r) => r._level == "crit"
)
|> endpoint(mapFn: (r) => ({ r with channel: "alerts", text: r.message, color: "danger"}))()
System-generated notification task

© 2020 InfluxData. All rights reserved. 23
*

Simple Custom Alerts
Send alerts without using InfluxDB
checks & notifications

© 2020 InfluxData. All rights reserved. 25
import "influxata/influxdb/tasks"
import "influxata/influxdb/slack"
import anomaly "contrib/anaisdg/anomalydetection"
toSlack = slack.endpoint(
url: "https://slack.com/api/chat.postMessage",
token: secrets.get(key: "SLACK_TOKEN")
)
from(bucket: "example")
|> range(start: tasks.lastSuccess(orTime: -task.every))
|> filter(fn: (r) => r._measurement == "m")
|> anomaly.mad()
|> filter(fn: (r) => r.level == "anomaly")
|> toSlack(
mapFn: (r) => { r with
channel: "anomalies",
text: "An anomaly has been detected on ${r.host}. ${r._field} is ${r._value}",
color: "danger"
}
)()
Custom anomaly detection alert

© 2020 InfluxData. All rights reserved. 26
import "influxata/influxdb/tasks"
import "influxata/influxdb/slack"
import anomaly "contrib/anaisdg/anomalydetection"
toSlack = slack.endpoint(
url: "https://slack.com/api/chat.postMessage",
token: secrets.get(key: "SLACK_TOKEN")
)
from(bucket: "example")
|> range(start: tasks.lastSuccess(orTime: -task.every))
|> filter(fn: (r) => r._measurement == "m")
|> anomaly.mad()
|> filter(fn: (r) => r.level == "anomaly")
|> toSlack(
mapFn: (r) => { r with
channel: "anomalies",
text: "An anomaly has been detected on ${r.host}. ${r._field} is ${r._value}",
color: "danger"
}
)()
Custom anomaly detection alert

© 2020 InfluxData. All rights reserved. 27
import "influxata/influxdb/tasks"
import "influxata/influxdb/slack"
import anomaly "contrib/anaisdg/anomalydetection"
toSlack = slack.endpoint(
url: "https://slack.com/api/chat.postMessage",
token: secrets.get(key: "SLACK_TOKEN")
)
from(bucket: "example")
|> range(start: tasks.lastSuccess(orTime: -task.every))
|> filter(fn: (r) => r._measurement == "m")
|> anomaly.mad()
|> filter(fn: (r) => r.level == "anomaly")
|> toSlack(
mapFn: (r) => { r with
channel: "anomalies",
text: "An anomaly has been detected on ${r.host}. ${r._field} is ${r._value}",
color: "danger"
}
)()
Custom anomaly detection alert

© 2020 InfluxData. All rights reserved. 28
import "influxata/influxdb/tasks"
import "influxata/influxdb/slack"
import anomaly "contrib/anaisdg/anomalydetection"
toSlack = slack.endpoint(
url: "https://slack.com/api/chat.postMessage",
token: secrets.get(key: "SLACK_TOKEN")
)
from(bucket: "example")
|> range(start: tasks.lastSuccess(orTime: -task.every))
|> filter(fn: (r) => r._measurement == "m")
|> anomaly.mad()
|> filter(fn: (r) => r.level == "anomaly")
|> toSlack(
mapFn: (r) => { r with
channel: "anomalies",
text: "An anomaly has been detected on ${r.host}. ${r._field} is ${r._value}",
color: "danger"
}
)()
Custom anomaly detection alert

© 2020 InfluxData. All rights reserved. 29
import "influxata/influxdb/tasks"
import "influxata/influxdb/slack"
import anomaly "contrib/anaisdg/anomalydetection"
toSlack = slack.endpoint(
url: "https://slack.com/api/chat.postMessage",
token: secrets.get(key: "SLACK_TOKEN")
)
from(bucket: "example")
|> range(start: tasks.lastSuccess(orTime: -task.every))
|> filter(fn: (r) => r._measurement == "m")
|> anomaly.mad()
|> filter(fn: (r) => r.level == "anomaly")
|> toSlack(
mapFn: (r) => { r with
channel: "anomalies",
text: "An anomaly has been detected on ${r.host}. ${r._field} is ${r._value}",
color: "danger"
}
)()
Custom anomaly detection alert

© 2020 InfluxData. All rights reserved. 30
import "influxata/influxdb/tasks"
import "influxata/influxdb/slack"
import anomaly "contrib/anaisdg/anomalydetection"
toSlack = slack.endpoint(
url: "https://slack.com/api/chat.postMessage",
token: secrets.get(key: "SLACK_TOKEN")
)
from(bucket: "example")
|> range(start: tasks.lastSuccess(orTime: -task.every))
|> filter(fn: (r) => r._measurement == "m")
|> anomaly.mad()
|> filter(fn: (r) => r.level == "anomaly")
|> toSlack(
mapFn: (r) => { r with
channel: "anomalies",
text: "An anomaly has been detected on ${r.host}. ${r._field} is ${r._value}",
color: "danger"
}
)()
Custom anomaly detection alert

© 2020 InfluxData. All rights reserved. 31
import "influxata/influxdb/tasks"
import "influxata/influxdb/slack"
import anomaly "contrib/anaisdg/anomalydetection"
toSlack = slack.endpoint(
url: "https://slack.com/api/chat.postMessage",
token: secrets.get(key: "SLACK_TOKEN")
)
from(bucket: "example")
|> range(start: tasks.lastSuccess(orTime: -task.every))
|> filter(fn: (r) => r._measurement == "m")
|> anomaly.mad()
|> filter(fn: (r) => r.level == "anomaly")
|> toSlack(
mapFn: (r) => { r with
channel: "anomalies",
text: "An anomaly has been detected on ${r.host}. ${r._field} is ${r._value}",
color: "danger"
}
)()
Custom anomaly detection alert
Execute generated function

Data Ingest

© 2020 InfluxData. All rights reserved. 33
import "csv"
csvData = "#datatype,string,long,dateTime:RFC3339,string,double
#group,false,false,false,true,false
#default,,,,,
,result,table,_time,region,_value
,,0,2018-05-08T20:50:00Z,east,15.43
,,0,2018-05-08T20:50:20Z,east,59.25
,,1,2018-05-08T20:50:00Z,west,62.73
"
csv.from(csv:csvData)
|> to(org: "my-org", bucket: "my-bucket")
Write data using annotated CSV

© 2020 InfluxData. All rights reserved. 34
import "experimental/csv"
csv.from(url: "https://influx-testdata.s3.amazonaws.com/noaa.csv")
|> to(org: "my-org", bucket: "noaa")
Write data using remote annotated CSV

© 2020 InfluxData. All rights reserved. 35
import "experimental/array"
data = [
{_time: 2018-05-08T20:50:00Z, region: "east", _value: 15.43},
{_time: 2018-05-08T20:50:20Z, region: "east", _value: 59.25},
{_time: 2018-05-08T20:50:00Z, region: "west", _value: 62.73}
]
array.from(rows: data)
|> to(org: "my-org", bucket: "my-bucket")
Write data with using an array of records

© 2020 InfluxData. All rights reserved. 36
// ...
response = http.get(
url: "http://localhost:8086/health",
headers: {Authorization: "Token mYsuP3rS3cRe7T0kEn"}
)
responseBody = response.body
seedRow = [{_time: now()}]
array.from(rows: seedRow)
|> map(fn: (r) => {
body = json.parse(data: responseBody)
return { r with
_measurement: "monitoring",
_field: "influxdb_status",
_value: body.message,
name: body.name
}
})
|> to(org: "my-org", bucket: "my-bucket")
Write data with using an HTTP response

© 2020 InfluxData. All rights reserved. 37
// ...
response = http.get(
url: "http://localhost:8086/health",
headers: {Authorization: "Token mYsuP3rS3cRe7T0kEn"}
)
responseBody = response.body
seedRow = [{_time: now()}]
array.from(rows: seedRow)
|> map(fn: (r) => {
body = json.parse(data: responseBody)
return { r with
_measurement: "monitoring",
_field: "influxdb_status",
_value: body.message,
name: body.name
}
})
|> to(org: "my-org", bucket: "my-bucket")
Write data with using an HTTP response

© 2020 InfluxData. All rights reserved. 38
// ...
response = http.get(
url: "http://localhost:8086/health",
headers: {Authorization: "Token mYsuP3rS3cRe7T0kEn"}
)
responseBody = response.body
seedRow = [{_time: now()}]
array.from(rows: seedRow)
|> map(fn: (r) => {
body = json.parse(data: responseBody)
return { r with
_measurement: "monitoring",
_field: "influxdb_status",
_value: body.message,
name: body.name
}
})
|> to(org: "my-org", bucket: "my-bucket")
Write data with using an HTTP response

© 2020 InfluxData. All rights reserved. 39

© 2020 InfluxData. All rights reserved. 40
import "experimental/prometheus"
prometheus.scrape(url: "http://localhost:8086/metrics")
|> to(
org: "my-org",
bucket: "my-bucket"
)
Scrape data from a Prometheus endpoint

© 2020 InfluxData. All rights reserved. 41
import "influxdata/influxdb/secrets"
import "sql"
username = secrets.get(key: "POSTGRES_USER")
password = secrets.get(key: "POSTGRES_PASS")
sql.from(
driverName: "postgres",
dataSourceName: "postgresql://${username}:${password}@localhost",
query:"SELECT * FROM example_table"
)
Query a SQL datasource

Data Enrichment

© 2020 InfluxData. All rights reserved. 43
join()

© 2020 InfluxData. All rights reserved. 44
_time sensorID _field _value
2020-06-01T00:12:00Z TM02001 temp 70.32
2020-06-01T00:12:01Z TM02002 temp 70.45
2020-06-01T00:12:02Z TM02003 temp 98.41
sensorID location status
TM02001 SF1.RM406 OK
TM02002 SF1.RM412 OK
TM02003 SF2.RM290 Requires service

© 2020 InfluxData. All rights reserved. 45
import "sql"
import "influxdata/influxdb/secrets"
username = secrets.get(key: "POSTGRES_USER")
password = secrets.get(key: "POSTGRES_PASS")
telemetry = from(bucket: "sensor-data")
|> range(start: -task.every)
|> filter(fn: (r) => r._measurement == "sensor-data" and r._field == "temp")
assets = sql.from(
driverName: "postgres",
dataSourceName: "postgresql://${username}:${password}@localhost",
query:"SELECT * FROM temp-sensors"
)
join(tables: {data: telemetry, assets: assets}, on: ["sensorID"])
|> to(org: "my-org", bucket: "enriched-sensor-data")
Enrich data with external data

© 2020 InfluxData. All rights reserved. 46
import "sql"
import "influxdata/influxdb/secrets"
username = secrets.get(key: "POSTGRES_USER")
password = secrets.get(key: "POSTGRES_PASS")
telemetry = from(bucket: "sensor-data")
|> range(start: -task.every)
|> filter(fn: (r) => r._measurement == "sensor-data" and r._field == "temp")
assets = sql.from(
driverName: "postgres",
dataSourceName: "postgresql://${username}:${password}@localhost",
query:"SELECT * FROM temp-sensors"
)
join(tables: {data: telemetry, assets: assets}, on: ["sensorID"])
|> to(org: "my-org", bucket: "enriched-sensor-data")
Enrich data with external data

© 2020 InfluxData. All rights reserved. 47
import "sql"
import "influxdata/influxdb/secrets"
username = secrets.get(key: "POSTGRES_USER")
password = secrets.get(key: "POSTGRES_PASS")
telemetry = from(bucket: "sensor-data")
|> range(start: -task.every)
|> filter(fn: (r) => r._measurement == "sensor-data" and r._field == "temp")
assets = sql.from(
driverName: "postgres",
dataSourceName: "postgresql://${username}:${password}@localhost",
query:"SELECT * FROM temp-sensors"
)
join(tables: {data: telemetry, assets: assets}, on: ["sensorID"])
|> to(org: "my-org", bucket: "enriched-sensor-data")
Enrich data with external data

© 2020 InfluxData. All rights reserved. 48
import "sql"
import "influxdata/influxdb/secrets"
username = secrets.get(key: "POSTGRES_USER")
password = secrets.get(key: "POSTGRES_PASS")
telemetry = from(bucket: "sensor-data")
|> range(start: -task.every)
|> filter(fn: (r) => r._measurement == "sensor-data" and r._field == "temp")
assets = sql.from(
driverName: "postgres",
dataSourceName: "postgresql://${username}:${password}@localhost",
query:"SELECT * FROM temp-sensors"
)
join(tables: {data: telemetry, assets: assets}, on: ["sensorID"])
|> to(org: "my-org", bucket: "enriched-sensor-data")
Enrich data with external data

© 2020 InfluxData. All rights reserved. 49
import "sql"
import "influxdata/influxdb/secrets"
username = secrets.get(key: "POSTGRES_USER")
password = secrets.get(key: "POSTGRES_PASS")
telemetry = from(bucket: "sensor-data")
|> range(start: -task.every)
|> filter(fn: (r) => r._measurement == "sensor-data" and r._field == "temp")
assets = sql.from(
driverName: "postgres",
dataSourceName: "postgresql://${username}:${password}@localhost",
query:"SELECT * FROM temp-sensors"
)
join(tables: {data: telemetry, assets: assets}, on: ["sensorID"])
|> to(org: "my-org", bucket: "enriched-sensor-data")
Enrich data with external data

© 2020 InfluxData. All rights reserved. 50
_time sensorID location status _field _value
2020-06-01T00:12:00Z TM02001 SF1 .RM406 OK temp 70.32
2020-06-01T00:12:01Z TM02002 SF1.RM412 OK temp 70.45
2020-06-01T00:12:02Z TM02003 SF2.RM290 Requires service temp 98.41

Data Transfer

© 2020 InfluxData. All rights reserved. 52

© 2020 InfluxData. All rights reserved. 53
import "influxdata/influxdb/secrets"
import "influxdata/influxdb/tasks"
cloudToken = secrets.get(key: "INFLUXDB_CLOUD_TOKEN")
from(bucket: "telemetry")
|> range(start: tasks.lastSuccess(orTime: -task.every))
|> aggregateWindow(every: 5m, fn: mean)
|> to(
host: "https://us-west-2-1.aws.cloud2.influxdata.com",
org: "my-cloud-org",
bucket: "fleet",
token: cloudToken,
)

© 2020 InfluxData. All rights reserved. 54

© 2020 InfluxData. All rights reserved. 55

© 2020 InfluxData. All rights reserved. 56

© 2020 InfluxData. All rights reserved. 57

© 2020 InfluxData. All rights reserved. 58
Other Use Cases
● Data sanitization
● Machine learning
● Reports
● Event triggers
● Data generation
● and much more!!

© 2020 InfluxData. All rights reserved. 59
Thank you!
Time for some Q&A