Package 'wasserportal'

Description

Helper function: base url for download

Usage

base_url_download()

Value

base url for download of csv/zip files prepared by R package

Description

Create Text Labels from Data Frame Columns

Usage

columns_to_labels(data, columns, fmt = "%s: %s", sep = ", ")

Arguments

Value

vector of character with as many elements as there are rows in data

Examples

data <- data.frame(number = 1:2, name = c("adam", "eva"), value = 3:4)
columns <- c("name", "value")
columns_to_labels(data, columns)
columns_to_labels(data, columns, fmt = "<p>%s: %s</p>", sep = "")

Description

The tables that appear in the API documentation of the wasserportal (https://wasserportal.berlin.de/download/wasserportal_berlin_getting_data.pdf) have been added to the wasserportal package. This function returns a list of data frames with each element representing one of these tables.

Usage

get_api_tables(name = NULL)

Arguments

Value

list of data frames or data frame specified by the name argument

Examples

get_api_tables()

Description

Get Daily Surfacewater Data: wrapper to scrape daily surface water data

Usage

get_daily_surfacewater_data(
  stations,
  variables = get_surfacewater_variables(),
  list2df = FALSE
)

Arguments

Value

list or data frame with all available data from Wasserportal

Examples

## Not run: 
stations <- wasserportal::get_stations()
variables <- wasserportal::get_surfacewater_variables()
variables
sw_data_daily <- wasserportal::get_daily_surfacewater_data(stations, variables)

## End(Not run)

Description

wrapper function to scrape all available raw data, i.e. groundwater level and quality data and save in list

Usage

get_groundwater_data(
  stations,
  groundwater_options = get_groundwater_options(),
  debug = TRUE,
  stations_list = NULL
)

Arguments

Value

list with elements "groundwater.level" and "groundwater.quality" data frames

Examples

## Not run: 
stations <- wasserportal::get_stations()
gw_data_list <- get_groundwater_data(stations)
str(gw_data_list)

## End(Not run)

Description

Helper function: get groundwater options

Usage

get_groundwater_options()

Value

return available groundwater data options and prepare for being used as input for get_groundwater_data

Examples

get_groundwater_options()

Description

Wasserportal Berlin: get overview options for stations

Usage

get_overview_options()

Value

list with shortcuts to station overview tables (⁠wasserportal.berlin.de/messwerte.php?anzeige=tabelle&thema=<shortcut>⁠)

Examples

get_overview_options()

Description

Helper function: get available station variables

Usage

get_station_variables(station_df)

Arguments

Value

returns names of available variables for station

Description

Get Stations

Usage

get_stations(
  type = c("list", "data.frame", "crosstable"),
  run_parallel = TRUE,
  n_cores = parallel::detectCores() - 1L,
  debug = TRUE
)

Arguments

Value

list with general station "overview" (either as list "overview_list" or as data.frame "overview_df") and a crosstable with information which parameters is available per station ("x" if available, NA if not)

Examples

stations <- wasserportal::get_stations(n_cores = 2L)
str(stations)

Description

Get Surface Water Quality for Multiple Monitoring Stations

Usage

get_surfacewater_qualities(station_ids, dbg = TRUE)

Arguments

Value

data frame with water quality data for multiple monitoring stations

Examples

## Not run: 
stations <- wasserportal::get_stations()
station_ids <- stations$overview_list$surface_water.quality$Messstellennummer
swq <- wasserportal::get_surfacewater_qualities(station_ids)
str(swq)

## End(Not run)

Description

Get Surface Water Quality for One Monitoring Station

Usage

get_surfacewater_quality(station_id)

Arguments

Value

data frame with water quality data for one monitoring station

Examples

## Not run: 
stations <- wasserportal::get_stations()
station_id <- stations$overview_list$surface_water.quality$Messstellennummer[1]
swq <- wasserportal::get_surfacewater_quality(station_id)
str(swq)

## End(Not run)

Description

Helper function: get surface water variables

Usage

get_surfacewater_variables()

Value

vector with surface water variables

Description

Wasserportal Berlin: get master data for a single station

Usage

get_wasserportal_master_data(master_url)

Arguments

Value

data frame with metadata for selected station

Examples

## Not run: 
stations_list <- wasserportal::get_stations(type = "list")

# GW Station
master_url <- stations_list %>%
  kwb.utils::selectElements("groundwater.level") %>%
  kwb.utils::selectColumns("stammdaten_link")[1L]

get_wasserportal_master_data(master_url)

# SW Station

# Reduce  to monitoring stations maintained by Berlin
master_urls <- stations_list %>%
  kwb.utils::selectElements("surface_water.water_level") %>%
  dplyr::filter(.data$Betreiber == "Land Berlin") %>%
  dplyr::pull(.data$stammdaten_link)

get_wasserportal_master_data(master_urls[1L])

## End(Not run)

Description

Wasserportal Berlin: get master data for a multiple stations

Usage

get_wasserportal_masters_data(master_urls, run_parallel = TRUE)

Arguments

Value

data frame with metadata for selected master urls

Examples

## Not run: 
stations_list <- wasserportal::get_stations(type = "list")

# Reduce  to monitoring stations maintained by Berlin
master_urls <- stations_list$surface_water.water_level %>%
  dplyr::filter(.data$Betreiber == "Land Berlin") %>%
  dplyr::pull(.data$stammdaten_link)

system.time(master_parallel <- get_wasserportal_masters_data(
  master_urls
))

system.time(master_sequential <- get_wasserportal_masters_data(
  master_urls,
  run_parallel = FALSE
))

## End(Not run)

Description

Get Names and IDs of the Stations of wasserportal.berlin.de

Usage

get_wasserportal_stations(type = "quality")

Arguments

Description

Wasserportal Berlin: get stations overview table

Usage

get_wasserportal_stations_table(
  type = get_overview_options()$groundwater$level,
  url_wasserportal = wasserportal_base_url()
)

Arguments

Value

data frame with master data of selected monitoring stations

Examples

types <- wasserportal::get_overview_options()
str(types)
sw_l <- wasserportal::get_wasserportal_stations_table(type = types$surface_water$water_level)
str(sw_l)

Description

Get Names and IDs of the Variables of wasserportal.berlin.de

Usage

get_wasserportal_variables(station = NULL)

Arguments

Description

Convenience helper for local debugging of the daily ZIP artefacts published at https://kwb-r.github.io/wasserportal. Downloads each ZIP, extracts the CSV, reads it with readr::read_csv() and prints a short summary (columns, row count, unique Messstellennummer count, head of the data). The intersection of Messstellennummer values across all loaded files is reported at the end so you can quickly see how many stations have measurements in every file.

Usage

inspect_gh_pages_zips(
  files = c("groundwater_level.zip", "groundwater_quality.zip"),
  base_url = "https://kwb-r.github.io/wasserportal",
  destdir = tempfile("wasserportal-inspect-"),
  head_rows = 5L
)

Arguments

Details

Returns the loaded data frames invisibly so the caller can further inspect them in R, e.g. dat$groundwater_level$Parameter |> table().

Value

invisibly a named list of tibbles, one per input file. Names are derived from the ZIP basename without the extension.

Examples

## Not run: 
# default: groundwater level + groundwater quality
dat <- inspect_gh_pages_zips()

# any ZIPs you want to inspect:
dat <- inspect_gh_pages_zips(files = c(
  "daily_surface-water_water-level.zip",
  "daily_surface-water_temperature.zip"
))

# keep the extracted CSVs:
dat <- inspect_gh_pages_zips(destdir = "~/tmp/wasserportal-inspect")

## End(Not run)

Description

Helper function: list data to csv or zip

Usage

list_data_to_csv_or_zip(data_list, file_prefix, to_zip)

Arguments

Value

loops through list of data frames and uses list names as filenames

Description

Helper function: list masters data to csv

Usage

list_masters_data_to_csv(masters_data_list)

Arguments

Value

loops through list of data frames and uses list names as filenames

Examples

## Not run: 
stations_list <- get_stations(type = "list")
masters_data_csv_files <- list_masters_data_to_csv(stations_list)
masters_data_csv_files

## End(Not run)

Description

Helper function: list timeseries data to zip

Usage

list_timeseries_data_to_zip(timeseries_data_list)

Arguments

Value

loops through list of data frames and uses list names as filenames

Examples

## Not run: 
stations <- wasserportal::get_stations()

# Groundwater Time Series
gw_tsdata_list <- wasserportal::get_groundwater_data(stations)
gw_tsdata_files <- wasserportal::list_timeseries_data_to_zip(gw_tsdata_list)

# Surface Water Time Series
sw_tsdata_list <- wasserportal::get_daily_surfacewater_data(stations)
sw_tsdata_files <- wasserportal::list_timeseries_data_to_zip(sw_tsdata_list)

## End(Not run)

Description

Helper function to read CSV

Usage

read(text, ...)

Arguments

Value

data frame with values

Description

This function downloads and reads CSV files from wasserportal.berlin.de.

Usage

read_wasserportal(
  station,
  variables = NULL,
  from_date = as.character(Sys.Date() - 90L),
  type = "single",
  include_raw_time = FALSE,
  stations_crosstable
)

Arguments

Details

The original timestamps (column timestamps_raw in the example below) are not all plausible, e.g. "31.03.2019 03:00" appears twice! They are corrected (column timestamp_corr) to represent a plausible sequence of timestamps in Berlin Normal Time (UTC+01) Finally, a valid POSIXct timestamp in timezone "Berlin/Europe" (UTC+01 in winter, UTC+02 in summer) is created, together with the additional information on the UTC offset (column UTCOffset, 1 in winter, 2 in summer).

Value

data frame read from the CSV file that the download provides. IMPORTANT: It is not yet clear how to interpret the timestamp, see example

Examples

## Not run: 
# Get a list of available water quality stations and variables
stations_crosstable <- wasserportal::get_stations(type = "crosstable")

# Set the start date
from_date <- "2021-03-01"

# Read the timeseries (multiple variables for one station)
water_quality <- wasserportal::read_wasserportal(
  station = stations_crosstable$Messstellennummer[1L],
  from_date = from_date,
  include_raw_time = TRUE,
  stations_crosstable = stations_crosstable
)

# Look at the first few records
head(water_quality)

# Check the metadata
#kwb.utils::getAttribute(water_quality, "metadata")

# Set missing values to NA
water_quality[water_quality == -777] <- NA

# Look at the first few records again
head(water_quality)

### How was the original timestamp interpreted?

# Determine the days at which summer time starts and ends, respectively
from_year <- as.integer(substr(from_date, 1L, 4L))
switches <- kwb.datetime::date_range_CEST(from_year)

# Reformat to dd.mm.yyyy
switches <- kwb.datetime::reformatTimestamp(switches, "%Y-%m-%d", "%d.%m.%Y")

# Define a pattern to look for timestamps "around" the switches
pattern <- paste(switches, "0[1-4]", collapse = "|")

# Look at the data for these timestamps
water_quality[grepl(pattern, water_quality$timestamp_raw), ]

# The original timestamps (timestamps_raw) were not all plausible, e.g.
# for March 2019. This seems to have been fixed by the "wasserportal"!
sum(water_quality$timestamp_raw != water_quality$timestamp_corr)

## End(Not run)

Description

Read Wasserportal Raw

Usage

read_wasserportal_raw(
  variable,
  station,
  from_date,
  type = "single",
  include_raw_time = FALSE,
  handle = NULL,
  stations_crosstable,
  api_version = 2L
)

Arguments

Value

????

Description

read_wasserportal_raw_gw

Usage

read_wasserportal_raw_gw(
  station = 149,
  stype = "gws",
  type = "single_all",
  from_date = "",
  include_raw_time = FALSE,
  handle = NULL,
  as_text = FALSE,
  dbg = FALSE
)

Arguments

Value

data.frame with values

Examples

## Not run: 
read_wasserportal_raw_gw(station = 149, stype = "gws")
read_wasserportal_raw_gw(station = 149, stype = "gwq")

## End(Not run)

Description

Read CSV File from Package's "extdata" Folder

Usage

readPackageFile(file, ...)

Arguments

Value

data frame representing the content of file

Description

Wipes historical telemetry rows from ThingsBoard for the given device and keys via DELETE /api/plugins/telemetry/DEVICE/{id}/timeseries/delete. Pass keys = NULL (the default) to clear every key the device currently knows – the function then calls tb_list_device_telemetry_keys first to discover them.

Usage

tb_delete_device_telemetry(
  device_id,
  keys = NULL,
  api_key = Sys.getenv("TB_API_KEY"),
  host = tb_default_host(),
  delete_latest = TRUE,
  username = Sys.getenv("TB_USERNAME"),
  password = Sys.getenv("TB_PASSWORD")
)

Arguments

Details

Server-side attributes (latitude, longitude, Bezirk, ...) and the device itself are NOT touched, only the time-series telemetry store. Re-running the demo push afterwards re-fills the cleared keys with the real Wasserportal timestamps.

Value

invisibly the number of keys submitted for deletion.

Examples

## Not run: 
id <- tb_get_device_id("wasserportal-gw-6038")
# wipe everything currently stored:
tb_delete_device_telemetry(id)
# wipe just the smoke-test GW-Stand value:
tb_delete_device_telemetry(id, keys = "GW-Stand")

## End(Not run)

Description

Lightweight read-only companion to tb_setup_devices: when you only need a device's internal UUID (e.g. to call the telemetry-delete endpoint), this returns it directly without touching the access token or creating the device on the side. Returns NA_character_ when the device does not exist.

Usage

tb_get_device_id(
  device_name,
  api_key = Sys.getenv("TB_API_KEY"),
  host = tb_default_host(),
  username = Sys.getenv("TB_USERNAME"),
  password = Sys.getenv("TB_PASSWORD")
)

Arguments

Value

device UUID (character) or NA_character_ if the lookup did not resolve.

Examples

## Not run: 
tb_get_device_id("wasserportal-gw-6038")

## End(Not run)

Description

Wraps GET /api/plugins/telemetry/DEVICE/{id}/keys/timeseries. Useful to discover what's actually in the device-side time-series store before a wipe, or to compare against the Parameter column of the gh-pages source data.

Usage

tb_list_device_telemetry_keys(
  device_id,
  api_key = Sys.getenv("TB_API_KEY"),
  host = tb_default_host(),
  username = Sys.getenv("TB_USERNAME"),
  password = Sys.getenv("TB_PASSWORD")
)

Arguments

Value

character vector of telemetry key names. May be of length 0.

Examples

## Not run: 
id <- tb_get_device_id("wasserportal-gw-6038")
tb_list_device_telemetry_keys(id)

## End(Not run)

Description

Calls POST /api/auth/login with a username/password pair and returns the JWT access token. This is the standard ThingsBoard REST API authentication and the only one available on self-hosted ThingsBoard Community Edition: unlike the account-level API key (a ThingsBoard Cloud convenience, generated under Account > Security > API keys), every edition – CE, PE and Cloud – accepts a username/password login.

Usage

tb_login(
  username = Sys.getenv("TB_USERNAME"),
  password = Sys.getenv("TB_PASSWORD"),
  host = tb_default_host()
)

Arguments

Details

The token is short-lived (ThingsBoard's default JWT expiration is 2.5 h), which is ample for the one-off device setup done by tb_setup_devices: the subsequent telemetry push uses the per-device access token, not this JWT, so no token refresh is implemented.

Transient failures (HTTP 408 / 429 / 500 / 502 / 503 / 504 and transport dropouts) are retried with exponential backoff, matching the predicate used for the telemetry POSTs so a flaky upstream does not abort the device-setup run on the first 5xx.

Value

the JWT access token as a character scalar, ready to be sent in an ⁠X-Authorization: Bearer <token>⁠ request header.

Credentials in error output

On a non-2xx response this helper surfaces an excerpt of the server's response body (via tb_error_body(), up to ~800 chars) in the R error message, and httr2::req_retry() prints retry messages to stderr. Stock ThingsBoard only echoes back the error description, not the request payload, so the password does not leak. If a self-hosted instance or reverse proxy is configured to echo request fields back in the error body, that excerpt would surface in R errors and – when captured with 2>&1 – in CI logs. Mask the relevant secrets in such environments.

Examples

## Not run: 
Sys.setenv(
  TB_HOST = "https://dashboards.inowas.org",
  TB_USERNAME = "[email protected]",
  TB_PASSWORD = "secret"
)
token <- tb_login()

## End(Not run)

Description

Wraps the per-device transport rate limits documented at https://thingsboard.io/docs/paas/eu/subscriptions/ into the parameters this package's push functions take. Pass the result into tb_push_station_telemetry() via mode, chunk_size and throttle_seconds so you stay below your plan's "Telemetry Transport messages/data points (Device)" thresholds.

Usage

tb_plan_defaults(plan = "free")

Arguments

Details

Across all paid PaaS tiers the per-device sustained limits are identical (2 000 telemetry data points per minute, 15 000 per hour), the only thing that changes is how aggressive a burst the platform tolerates before it drops a request. Free additionally rejects the bulk array form on the device telemetry endpoint, so its default is mode = "single".

Self-hosted ThingsBoard CE has no per-tenant rate limit by default, hence the much larger chunk size and zero throttle.

Value

named list with mode, chunk_size and throttle_seconds, ready to be spread into tb_push_station_telemetry().

Examples

tb_plan_defaults("free")
tb_plan_defaults("free-bulk")
tb_plan_defaults("ce")

Description

Sends a flat ⁠{"key": value, ...}⁠ JSON object to the ThingsBoard telemetry endpoint, letting the server stamp it with the current time. This is the simplest possible telemetry POST and is useful both as a smoke test for the device-token auth path and as a fallback when the bulk-with-ts format is rejected (some ThingsBoard Cloud Maker tier configurations return an opaque HTTP 500 to the array-of-records form even though the same device accepts attributes and latest-style single records).

Usage

tb_push_latest_telemetry(
  values,
  device_token,
  host = tb_default_host()
)

Arguments

Value

invisibly the number of keys that were sent.

Examples

## Not run: 
tb_push_latest_telemetry(
  values = list(`GW-Stand` = 35.6, `Wassertemperatur` = 11.2),
  device_token = Sys.getenv("TB_DEVICE_TOKEN")
)

## End(Not run)

Description

Sends station metadata (coordinates, level reference, operator, ...) as client-side attributes to the ThingsBoard device attributes endpoint (⁠/api/v1/{token}/attributes⁠). Attributes are key/value pairs without a timestamp; ThingsBoard overwrites the previous value on every push.

Usage

tb_push_station_attributes(
  attributes,
  device_token,
  host = tb_default_host()
)

Arguments

Value

invisibly the number of attributes that were sent.

Examples

## Not run: 
tb_push_station_attributes(
  attributes = list(
    name = "Pegel Mueggelheim",
    latitude = 52.4291,
    longitude = 13.6450,
    pegelnullpunkt_m_NHN = 32.18
  ),
  device_token = Sys.getenv("TB_DEVICE_TOKEN_5867000")
)

## End(Not run)

Description

Sends long-format measurements of a single Wasserportal monitoring station to the device telemetry endpoint of a ThingsBoard instance (⁠/api/v1/{token}/telemetry⁠). Works against ThingsBoard Cloud (e.g. the free Maker tier on ⁠https://thingsboard.cloud⁠), self-hosted ThingsBoard Community Edition and ⁠https://demo.thingsboard.io⁠ since the device-token API is identical on all of them.

Usage

tb_push_station_telemetry(
  data,
  device_token,
  ts_col = "Datum",
  value_col = "Messwert",
  key_col = "Parameter",
  single_key = "value",
  host = tb_default_host(),
  chunk_size = 100L,
  mode = c("single", "bulk"),
  throttle_seconds = NULL,
  max_active = 10L,
  verbose = TRUE
)

Arguments

Details

Long-format input is pivoted on the fly so that every distinct value of key_col becomes one telemetry key inside ThingsBoard, sharing the same timestamp.

Value

invisibly the number of telemetry timestamps that were sent.

Examples

## Not run: 
stations <- wasserportal::get_stations()
gw <- wasserportal::get_groundwater_data(stations)
one_station <- dplyr::filter(
  gw$groundwater.level,
  .data$Messstellennummer == "149"
)
tb_push_station_telemetry(
  data = one_station,
  device_token = Sys.getenv("TB_DEVICE_TOKEN_149")
)

## End(Not run)

Description

Convenience wrapper for the initial setup against a fresh ThingsBoard tenant. Authenticates with either a username/password login (JWT – works on every ThingsBoard edition, required for self-hosted Community Edition) or an account-level API key (ThingsBoard Cloud), then:

1. Create one device per name (or fetch the device if it already exists),

2. Read each device's access token via the credentials endpoint.

The returned named character vector can be fed directly into tb_push_station_telemetry as device_token.

Usage

tb_setup_devices(
  station_ids,
  api_key = Sys.getenv("TB_API_KEY"),
  host = tb_default_host(),
  name_prefix = "wasserportal-",
  device_type = "wasserportal",
  username = Sys.getenv("TB_USERNAME"),
  password = Sys.getenv("TB_PASSWORD")
)

Arguments

Details

Set TB_USERNAME + TB_PASSWORD for the login route, or generate an API key in the ThingsBoard Cloud UI under Account > Security > API keys > Generate and set TB_API_KEY.

Value

named character vector. Names are the input station_ids, values are device access tokens.

Examples

## Not run: 
# Self-hosted ThingsBoard Community Edition (username/password login):
Sys.setenv(
  TB_HOST = "https://dashboards.inowas.org",
  TB_USERNAME = "[email protected]",
  TB_PASSWORD = "secret"
)
tokens <- tb_setup_devices(c("149", "5867000", "5803900"))

# ThingsBoard Cloud (account API key):
Sys.setenv(
  TB_HOST = "https://eu.thingsboard.cloud",
  TB_API_KEY = "<paste-your-API-key-here>"
)
tokens <- tb_setup_devices(c("149", "5867000", "5803900"))

## End(Not run)

Description

Helper function: Base Url of Berlin Wassersportal

Usage

wasserportal_base_url()

Value

string with base url of Berlin Wasserportal

Description

Wasserportal Master Data: download and Import in R List

Usage

wp_masters_data_to_list(
  overview_list_names,
  target_dir = tempdir(),
  file_prefix = "stations_",
  is_zipped = FALSE
)

Arguments

Value

downloads csv master data from Wasserportal

Examples

## Not run: 
overview_list_names <- names(wasserportal::get_stations(type = "list"))
wp_masters_data_list <- wp_masters_data_to_list(overview_list_names)

## End(Not run)

Description

Wasserportal Time Series Data: download and Import in R List

Usage

wp_timeseries_data_to_list(
  overview_list_names,
  target_dir = tempdir(),
  is_zipped = TRUE
)

Arguments

Value

downloads (zipped) data from wasserportal

Examples

## Not run: 
overview_list_names <- names(wasserportal::get_stations(type = "list"))
wp_timeseries_data_list <- wp_timeseries_data_to_list(overview_list_names)

## End(Not run)