# Install devtools, if required, from CRAN
# install.packages("devtools")

# Install kwb.monitoring from GitHub
devtools::install_github("kwb-r/kwb.monitoring", dependencies = TRUE)

library(kwb.monitoring)

# Create a test directory structure for raw data
flow_dir <- getOrCreatePath("FLOW_DIR", dictionary)

last_flow_dir <- kwb.utils::createDirectory(file.path(flow_dir, "20180601"))
#> The parent directory /tmp/RtmpkzsQ13/Rinstf9f4a11a94c/kwb.monitoring/extdata/rawdata/test-project/KWB/flows needs to be created first.
#> The parent directory /tmp/RtmpkzsQ13/Rinstf9f4a11a94c/kwb.monitoring/extdata/rawdata/test-project/KWB needs to be created first.
#> The parent directory /tmp/RtmpkzsQ13/Rinstf9f4a11a94c/kwb.monitoring/extdata/rawdata/test-project needs to be created first.
#> The directory "/tmp/RtmpkzsQ13/Rinstf9f4a11a94c/kwb.monitoring/extdata/rawdata/test-project" was created.
#> The directory "/tmp/RtmpkzsQ13/Rinstf9f4a11a94c/kwb.monitoring/extdata/rawdata/test-project/KWB" was created.
#> The directory "/tmp/RtmpkzsQ13/Rinstf9f4a11a94c/kwb.monitoring/extdata/rawdata/test-project/KWB/flows" was created.
#> The directory "/tmp/RtmpkzsQ13/Rinstf9f4a11a94c/kwb.monitoring/extdata/rawdata/test-project/KWB/flows/20180601" was created.

file <- file.path(last_flow_dir, "flows.csv")

writeLines(con = file, c(
  "NIVUS",
  "CPU32",
  "Datum",
  "Fenster min",
  "Fenster max",
  paste0("skip_", 6:8),
  "Datum\tUhrzeit\tH\tv\tQ\tT",
  "01.06.2018\t00:00:00\t0.1\t0.1\t0.1\t0.1", 
  "01.06.2018\t00:01:00\t0.2\t0.1\t0.1\t0.1",
  "01.06.2018\t00:02:00\t0.3\t0.1\t0.1\t0.1", 
  "01.06.2018\t00:03:00\t0.2\t0.1\t0.1\t0.1",
  "01.06.2018\t00:04:00\t0.1\t0.1\t0.1\t0.1", 
  "01.06.2018\t00:05:00\t0.0\t0.1\t0.1\t0.1"
))

kwb.utils::createDirectory(getOrCreatePath("SAMPLE_DIR", dictionary))
#> The directory "/tmp/RtmpkzsQ13/Rinstf9f4a11a94c/kwb.monitoring/extdata/rawdata/test-project/KWB/samples" was created.
#> [1] "/tmp/RtmpkzsQ13/Rinstf9f4a11a94c/kwb.monitoring/extdata/rawdata/test-project/KWB/samples"

# read_hydraulics --------------------------------------------------------------
read_hydraulics <- function(settings)
{
  flowDirectory <- getOrCreatePath("FLOW_DIR", dictionary = settings$dictionary)
  
  flowSubdirCurrent <- getCurrentFlowSubDirectory(flowDirectory)
  
  csv <- getOrCreatePath(
    "FLOW_CSV", 
    dictionary = settings$dictionary, 
    FLOW_SUBDIR_CURRENT = flowSubdirCurrent
  )
  
  hydraulics <- kwb.logger::readLogger_NIVUS_PCM4_2(kwb.utils::safePath(csv))
  
  renames <- list(
    myDateTime = "DateTime",
    Fuellstand_m = "H",
    Geschw_m_s = "v",
    Durchfluss_l_s = "Q",
    T_.C = "T"
  )
  
  columnNames <- as.character(renames)
  
  hydraulics <- kwb.utils::selectColumns(
    kwb.utils::renameColumns(hydraulics, renames), 
    columnNames
  )
  
  hydraulics$DateTime <- as_utc(hydraulics$DateTime)
  
  hydraulics
  ### data frame with columns "DateTime" (POSIXct, UTC), "H",
  ### "v", "Q", "T"
}

# getCurrentFlowSubDirectory ---------------------------------------------------
getCurrentFlowSubDirectory <- function
(
  flowDirectory
)
{
  subDirectories <- dir(flowDirectory)
  
  patternSubdir <- "^\\d{8}$"
  unexpected <- grep(patternSubdir, subDirectories, value=TRUE, invert=TRUE)
  
  if (! kwb.utils::isNullOrEmpty(unexpected)) {
    stop(
      sprintf("There are unexpected files/folders in \"%s\": %s\n%s\n",
              flowDirectory, paste(kwb.utils::hsQuoteChr(unexpected)), 
              "Only folders of type 'YYYYMMDD' expected!")
    )
  }
  
  currentSubdir <- sort(setdiff(subDirectories, unexpected), decreasing=TRUE)[1]
  
  if (is.na(currentSubdir)) {
    warning("There is no subdirectory 'YYYYMMDD' in ", flowDirectory)
  }
  else {
    cat("Most recent flow sub-directory:", currentSubdir, "\n")
  }  
  
  return(currentSubdir)
}

# readSamplerFile --------------------------------------------------------------
readSamplerFile <- function 
(
  samplerFile, bottlesToConsider, siteCode = NA
)  
{
  cat(sprintf("Reading sample data from \"%s\"... ", basename(samplerFile)))
  
  if (containsNulString(samplerFile)) {
    cat("skipped!\n")
    warning(paste(
      sprintf(
        "File \"%s\" contains null string and is skipped!",
        basename(samplerFile)),
      "Remove first two bytes of the file!"))
    sampleDataExtended <- NULL
  }
  else {    
    sampleData <- kwb.logger::readLogger_SIGMA_SD900(samplerFile)
    cat("ok.\n")
    sampleSite <- attr(sampleData, "metadata")$SITE_ID
    
    if (!is.na(siteCode) && sampleSite != siteCode) {
      stop(sprintf("The SITE_ID indicated in \"%s\" is not \"%s\" as expected!",
                   sampleSite, siteCode))
    }
    
    sampleDataExtended <- cbind(
      samplerFile = basename(samplerFile), 
      sampleData, 
      stringsAsFactors = FALSE
    )
  }
  
  # filter for relevant bottles
  if (!is.null(bottlesToConsider) & 
        ! all(is.na(bottlesToConsider))) {
    cat("Filtering for bottles", commaCollapsed(bottlesToConsider), "... ")
    indices <- which(sampleDataExtended$bottle %in% bottlesToConsider)
    sampleDataExtended <- sampleDataExtended[indices, ]
    cat("ok.\n")
  }
  
  kwb.utils::renameColumns(sampleDataExtended, list(myDateTime = "sampleTime"))
}

# rainGaugesCorrelatedWithStation ----------------------------------------------
rainGaugesCorrelatedWithStation <- function 
### selects gauges after the result of the volume correlation test
(
  station = NULL
)
{
  x <- list(
    ALT = c("ReiI", "BlnXI", "BlnIX", "BlnX"),
    NEU = c("Wit", "ReiI", "BlnIX")    
  )
  
  if (!is.null(station)) {
    x <- x[[station]]
  } 
  
  x
  ### list (one list element per monitoring site) of character vectors 
  ### representing rain gauge names
}

# processSampleEvent -----------------------------------------------------------
processSampleEvent <- function
(
  hydraulicData, 
  settings, 
  events, 
  eventsAndStat, 
  sampleEventIndex = -1, 
  to.pdf = TRUE,
  verbose = FALSE
) 
{  
  stopifnot(length(sampleEventIndex) == 1)
  
  # Filter for sampler event by index
  fileName <- getByPositiveOrNegativeIndex(
    elements = sort(unique(events$samplerEvents$samplerFile)), 
    index = sampleEventIndex
  ) 
  
  sampleInformation <- filterSampleEventsForFilename(
    events = events, 
    fileName = fileName
  )  
  
  samplerEvent <- sampleInformation$samplerEvents
  
  if (verbose) {
    printSampleInformation(sampleInformation)
  }
  
  saveSampleInformation(sampleInformation, settings, sampleFile=samplerEvent$samplerFile)
  
  # find "merged" event(s) containing the "sampler event"
  mergedEventNumber <- indicesOfEventsContainingEvent(eventsAndStat$merged, samplerEvent)
  
  stopifnot(length(mergedEventNumber) == 1)
  
  mergedEventAndStat <- eventsAndStat$merged[mergedEventNumber, ]
  
  # select subset of data within the sampler event
  hydraulicSubset <- hsGetEvent(hydraulicData, events = mergedEventAndStat, 1)
  
  if (kwb.utils::isNullOrEmpty(hydraulicSubset)) {
    warning("There is no hydraulic data for this event:\n", 
            formatEvent(mergedEventAndStat))
    return()
  }
  
  # generate a column containing the bottle number for each timestep  
  hydraulicSubset$bottle <- hsEventNumber(
    hydraulicSubset$DateTime, 
    events = sampleInformation$bottleEvents,
    eventNumbers=sampleInformation$bottleEvents$bottle
  )
  
  # generate a column containing the cumulative volume
  hydraulicSubset$Vcum_m3 <- cumsum(hydraulicSubset$Q)*settings$tstep.fill.s/1000
  
  # write interpolated data to files for "quality assurance"
  eventName <- sampleLogFileToSampleName(samplerEvent$samplerFile)
  
  writeCsvToPathFromDictionary(
    dataFrame = hydraulicSubset, 
    key = "SAMPLED_EVENT_CSV_HYDRAULICS", 
    SAMPLED_EVENT_NAME = eventName, 
    settings = settings, 
    open.directory = FALSE
  )
  
  # calculate sum of flows per bottle  
  volumeCompositeSample <- calculateVolumeCompositeSample(  
    hydraulicSubset, settings
  )
  
  writeCsvToPathFromDictionary(
    dataFrame = addSumRow(volumeCompositeSample), 
    key = "SAMPLED_EVENT_CSV_COMPOSITE", 
    SAMPLED_EVENT_NAME = eventName,
    settings = settings, 
    open.directory = FALSE
  )
  
  plot_sampled_event(
    hydraulicData = hydraulicData,
    settings = settings, 
    sampleInformation = sampleInformation, 
    mergedEventAndStat = mergedEventAndStat,
    volumeCompositeSample = volumeCompositeSample,
    to.pdf = to.pdf
  )  
}

# Read last available hydraulic data (required if station changed)
# "-10d" = last 10 days
hydraulicData.raw <- kwb.base::selectTimeInterval(
  read_hydraulics(settings), width = "-30d"
)
#> Most recent flow sub-directory: 20180601 
#> 
#> *** selectTimeInterval: t1=2018-05-02 00:05:00, t2=2018-06-01 00:05:00, width=-30d

writeCsvToPathFromDictionary(
  hydraulicData.raw, key = "HYDRAULIC_DATA_RAW", settings, 
  open.directory = FALSE
)
#> Writing to '/tmp/RtmpkzsQ13/Rinstf9f4a11a94c/kwb.monitoring/extdata/rawdata/flows_raw.csv'... ok.

# Show overview plots (complete and daily), optional
showOverview(
  hydraulicData.raw, settings, Qmax = 20, Hmax = 35, to.pdf = TRUE, 
  save.pdf = TRUE
)

# Prepare hydraulic data (fill gaps, interpolate/predict)
hydraulicData.all <- validateAndFillHydraulicData(
  hydraulicData = hydraulicData.raw, 
  settings = settings
)
#> There are 2 timestamps (out of a total of 5 ) that are not multiples of the timestep ( 120 seconds ):
#> [1] "2018-06-01 00:01:00 UTC" "2018-06-01 00:03:00 UTC"
#> replaceMissingQMethod = interpolate -> The interpolated values are used when Q.raw is NA.

# Write data to file (optional)
writeCsvToPathFromDictionary(
  hydraulicData.all, key = "HYDRAULIC_DATA_VAL", settings, 
  open.directory = FALSE
)
#> Writing to '/tmp/RtmpkzsQ13/Rinstf9f4a11a94c/kwb.monitoring/extdata/rawdata/flows_val.csv'... ok.

hydraulicData <- removeIntervals(
  hydraulicData.all, 
  intervals = intervalsToRemove[[settings$station]]
)

# Build different kinds of events (hydraulic, sample, merged), required
events <- getAllTypesOfEvents(
  hydraulicData, settings, FUN.readSamplerFile = readSamplerFile
)
#> Warning in getHydraulicEvents(hydraulicData, settings): *** The event conditions are never met.
#>   Range of Q values: [0.100000, 0.100000]
#>   Range of H values: [0.100000, 0.300000]
#>   Q-threshold: 3.000000
#>   H-threshold: 0.055000
#> Warning: There are no hydraulic events according to the defined criteria (or
#> hydraulicData was NULL)!
#> Warning: There are no sample files (matching "sample_\\d.csv") in:
#>   "/tmp/RtmpkzsQ13/Rinstf9f4a11a94c/kwb.monitoring/extdata/rawdata/test-project/KWB/samples"
#> Warning: There are no sampling events!
#> Warning: No hydraulicEvents given to getMergedEvents(). Returning
#> samplerEvents.

# Plot overview of event limits (optional)
# plotEventOverview(events[c("hydraulic", "samplerEvents", "merged")], settings)
# 
# # Write events to files (optional)
# writeCsvToPathFromDictionary(events$samplingEvents, "SAMPLING_EVENTS", settings, 
#                              open.directory = FALSE)
# writeCsvToPathFromDictionary(events$bottleEvents, "BOTTLE_EVENTS", settings, 
#                              open.directory = FALSE)
# writeCsvToPathFromDictionary(events$samplerEvents, "SAMPLER_EVENTS", settings, 
#                              open.directory = FALSE)
# 
# # Add statistics to hydraulic events
# eventsAndStat <- addStatisticsToEvents(events, hydraulicData.all)
# 
# # Write event including statistics to file (optional)
# writeCsvToPathFromDictionary(
#   eventsAndStat$hydraulic, "HYDRAULIC_EVENTS", settings, 
#   open.directory = FALSE
# )
# 
# # Load rain data if available
# if (file.exists(rainDataFile)) {
#   load(rainDataFile)
# } else {
#   rainData <- NULL #(if you dont want to show rain data)    
# }
# 
# seriesNames <- rainGaugesCorrelatedWithStation(station = settings$station)
# 
# hydraulicEvents <- eventsAndStat$hydraulic
# 
# Vt <- settings$Vthresholds[settings$station]
# 
# hydraulicEvents <- kwb.utils::renameColumns(
#   hydraulicEvents, list(event = "eventNumber")
# )
# 
# # Select events with a minimum volume
# selected <- !is.na(hydraulicEvents$V.m3) & hydraulicEvents$V.m3 > Vt
# events.to.plot <- hydraulicEvents[selected,]
# 
# # plot hydraulic events with rain
# plot_hydraulic_events(
#   hydraulicData = hydraulicData, 
#   settings = settings, 
#   eventsAndStat = events.to.plot,
#   to.pdf = TRUE, 
#   rainData = rainData, 
#   gauges = seriesNames[1:3]
# )  
# 
# # Analyse one sampler event given by its number ("sampleEventIndex")
# # according to the order in the sample log files. By giving a negative index
# # you may choose events from the end (-1 means "last", -2 one before last,
# # etc.)
# processSampleEvent(
#   hydraulicData, settings, events, eventsAndStat, sampleEventIndex = -2, 
#   to.pdf = FALSE
# )

# Find H-Q regression models ---------------------------------------------------
if (FALSE) 
{ 
  # Read last available hydraulic data (required if station changed)
  # "-10d" = last 10 days
  hydraulicData.raw <- kwb.base::selectTimeInterval(read_hydraulics(settings), width="-30d")
  
  #Remove bad Intervals
  hydraulicData <- removeIntervals(
    hydraulicData.raw, 
    intervals = intervalsToRemove[[settings$station]]
  )
  
  # run "regression finder" with a subset of the raw data
  selectIntervalsForCorrelation(kwb.base::selectTimeInterval(hydraulicData, width="-30d"), settings)
  
  # the model last applied is now available in the variable "regresssionState"
  saveRegressionModel(regressionState$model, settings)
}