Pilot: Ultimate

Define Login Credentials

For writing to InfluxDBCloud you need to run usethis::edit_r_environ() and define at least the following environment variables:

###############################################################################
### For InfluxDB-Cloud
###############################################################################
ULTIMATE_INFLUXDB_URL = "<ultimate_influxdbcloud_url>"
# InfluxDB API-Token with Write for bucket "ultimate"
ULTIMATE_INFLUXDB_TOKEN = "<ultimate_influxdbcloud_apitoken>"
ULTIMATE_INFLUXDB_ORG = "<ultimate_influxdbcloud_organisation>"

################################################################################  
## Optional (for downloading raw data from Nextcloud)
################################################################################
NEXTCLOUD_URL = "https://<replace-with-nextcloud-cloud-url>"
NEXTCLOUD_USER = "<your-nextcloud-username>" # your username
NEXTCLOUD_PASSWORD = "your-nextcloud-app-password" ### see details below

#3 For creating <your-nextcloud-app-password>:
#3.1 go to: https://replace-with-nextcloud-url/index.php/settings/user/security
#3.2 scroll down to create new app password
#3.3 select a name e.g. r-script and copy the token and replace your-nextcloud-app-password

Finally you need to restart Rstudio and proceed with the code below:

Install R package

# Enable repository from kwb-r
options(repos = c(
  kwbr = 'https://kwb-r.r-universe.dev',
  CRAN = 'https://cloud.r-project.org'))

# Download and install kwb.pilot in R
install.packages('kwb.pilot')

Define Paths

library(kwb.pilot)

paths_list <- list(
  root_ci = tempdir(),
  root_windows = "C:/kwb",
  root = "",
  common_root = "projects/ultimate/raw_data_pilots",
  data_raw = "data", 
  data_imp = "data_imported",
  a_site_code = "Pilot_A",
  b_site_code = "Pilot_B",
  a_cloud = "<common_root>/<a_site_code>",
  a_cloud_raw = "<a_cloud>/<data_raw>", 
  a_cloud_imported = "<a_cloud>/<data_imp>",
  a_local = "<root>/<a_cloud>",
  a_local_raw = "<a_local>/<data_raw>", 
  a_local_imported = "<a_local>/<data_imp>",
  b_cloud = "<common_root>/<b_site_code>",
  b_cloud_raw = "<b_cloud>/<data_raw>", 
  b_cloud_imported = "<b_cloud>/<data_imp>",
  b_local = "<root>/<b_cloud>",
  b_local_raw = "<b_local>/<data_raw>", 
  b_local_imported = "<b_local>/<data_imp>"
)

paths <- kwb.utils::resolve(paths_list, 
                            root = ifelse(identical(Sys.getenv("CI"), 
                                                            "true"),
                                                  paths_list$root_ci, 
                                                  paths_list$root_windows)
                            )

Write to InfluxDBCloud


################################################################################
### 1. Write Pilot Plant Raw Data to InfluxDB Cloud
################################################################################

################################################################################
### 1.1 Pilot A
################################################################################

## Step: kwb.pilot::download_nextcloud_files() is optional and can be skipped
## if the data is provided in the `paths$raw_data_dir`
files_pilot_a <- kwb.pilot::download_nextcloud_files(dir_cloud = paths$a_cloud_raw,
                                    dir_local = paths$a_local_raw)


if(length(files_pilot_a) > 0) {

tsv_paths <- list.files(
  path = paths$a_local_raw,
  full.names = TRUE,
  pattern = "xls$"
)

kwb.pilot::write_to_influxdb_loop(
  tsv_paths = tsv_paths,
  paths = list(site_code = paths$a_site_code,
               raw_data_dir = paths$a_local_raw),
  changed_only = FALSE,
  max_tsv_files = 10,
  batch_size = 5000
)


### Move all files for Pilot A`s cloud "raw" data directory to the "imported" 
### directory (in case that a file is already existing there: overwrite it)!
paths_pilot_a <- data.frame(raw = file.path(paths$a_cloud_raw, files_pilot_a),
                            imported = file.path(paths$a_cloud_imported, files_pilot_a)
                            )

kwb.pilot::move_nextcloud_files(paths_pilot_a, overwrite = TRUE)
} else {
  message(sprintf("Nothing to do for '%s'! No new files in '%s'",
                  paths$a_site_code,
                  paths$a_cloud_raw))
}


################################################################################
### 1.2 Pilot B
################################################################################

## Step: kwb.pilot::download_nextcloud_files() is optional and can be skipped
## if the data is provided in the `paths$raw_data_dir`
files_pilot_b <- kwb.pilot::download_nextcloud_files(dir_cloud = paths$b_cloud_raw,
                                    dir_local = paths$b_local_raw)

if(length(files_pilot_b) > 0) {
tsv_paths <- list.files(
  path = paths$b_local_raw,
  full.names = TRUE,
  pattern = "xls$"
)

kwb.pilot::write_to_influxdb_loop(
  tsv_paths = tsv_paths,
  paths = list(site_code = paths$b_site_code,
               raw_data_dir = paths$b_local_raw),
  changed_only = FALSE,
  max_tsv_files = 5,
  batch_size = 5000
)

### Move all files for Pilot B`s cloud "raw" data directory to the "imported" 
### directory (in case that a file is already existing there: overwrite it)!
paths_pilot_b <- data.frame(raw = file.path(paths$b_cloud_raw, files_pilot_b),
                            imported = file.path(paths$b_cloud_imported, files_pilot_b)
                            )

kwb.pilot::move_nextcloud_files(paths_pilot_b, overwrite = TRUE)
} else {
  message(sprintf("Nothing to do for '%s'! No new files in '%s'",
                  paths$b_site_code,
                  paths$b_cloud_raw))
}