Workflow

## Original Endnote data
refs_org <- kwb.endnote::create_references_df(endnote_list)

## "Cleaned" Endnote data
refs_clean <- kwb.endnote::clean_references_df(endnote_list)
#> Check 'Author Names' ... No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> No cleaning of author_names implemented yet. Only hints are generated in
#>     case that user defines 'give_hints = TRUE' (default: FALSE) ... ok. (0.00 secs) 
#> ok. (0.01 secs) 
#> Clean 'DOI' ... ok. (0.00 secs) 
#> Clean 'Project Names' ... ok. (0.00 secs) 
#> Clean 'Accessibility' ... ok. (0.00 secs)

## Check both for differences
diffs_df <- kwb.endnote::check_for_differences(refs_org, refs_clean)
#> Tidying data.frame 'refs_org' and rename 'value' to 'value_refs_org'. Saving to refs_org_tidy ... ok. (0.01 secs) 
#> Tidying data.frame 'refs_clean' and rename 'value' to 'value_refs_clean'. Saving to refs_clean_tidy ... ok. (0.01 secs) 
#> Joining data.frame 'refs_org_tidy' and 'refs_clean_tidy' with cols ''rec_number', 'key'' ... ok. (0.00 secs) 
#> Identifying indices with different content for 'value_refs_org' and 'value_refs_clean' ... ok. (0.21 secs) 
#> Content has changed for 393 indices

diffs_df %>%
  dplyr::filter(.data$key != "abstract") %>% 
  head() %>% 
  knitr::kable()

en_old_xml <- kwb.endnote::extdata_file("2020-05-25_KWB-documents.xml")
en_new_xml <- kwb.endnote::extdata_file("2020-06-17_KWB-documents.xml")

en_old_list <- kwb.endnote::create_endnote_list(en_old_xml)
en_new_list <- kwb.endnote::create_endnote_list(en_new_xml)

en_old_df <- kwb.endnote::create_references_df(en_old_list)
en_new_df <- kwb.endnote::create_references_df(en_new_list)

en_diffs_df <- kwb.endnote::check_for_differences(en_old_df, en_new_df)
#> Tidying data.frame 'en_old_df' and rename 'value' to 'value_en_old_df'. Saving to en_old_df_tidy ... ok. (0.02 secs) 
#> Tidying data.frame 'en_new_df' and rename 'value' to 'value_en_new_df'. Saving to en_new_df_tidy ... ok. (0.02 secs) 
#> Joining data.frame 'en_old_df_tidy' and 'en_new_df_tidy' with cols ''rec_number', 'key'' ... ok. (0.01 secs) 
#> Identifying indices with different content for 'value_en_old_df' and 'value_en_new_df' ... ok. (0.23 secs) 
#> Content has changed for 1092 indices

en_diffs_df %>% 
  dplyr::filter(.data$key != "abstract") %>% 
  dplyr::arrange(dplyr::desc(.data$rec_number), .data$key) %>% 
  knitr::kable()

n_new_entries <- length(unique(en_diffs_df$rec_number))
  

multi_cols_list <- kwb.endnote::tidy_multi_cols_list(refs_clean)

names(multi_cols_list)
#> [1] "author"           "author_secondary" "author_tecondary" "urls_pdf"        
#> [5] "keywords"


hints_author_names <- tibble::tibble(author = unique(multi_cols_list$author$value)) %>%
  dplyr::mutate(hints = kwb.endnote::give_hints_author_names(.data$author)) %>% 
  dplyr::filter(stringr::str_detect(.data$hints, pattern = "^fix_")) %>% 
  dplyr::arrange(.data$author) 
#> Generate hints for 'Author Names' ... Replacing 13 values with fix_author_name_by_deleting_last_semicolon ... Replacing 6 values with fix_firstname_lastname_with_lastname_semicolon_firstname ... Replacing 13 values with fix_multiple_authors_per_line ... ok. (0.00 secs) 
#> ok. (0.00 secs) 
#> ok. (0.00 secs) 
#> ok. (0.03 secs)


hints_author_names 
#> # A tibble: 19 × 2
#>    author                  hints                                                
#>    <chr>                   <chr>                                                
#>  1 "Adrien Moreau"         fix_firstname_lastname_with_lastname_semicolon_first…
#>  2 "Andreas Matzinger"     fix_firstname_lastname_with_lastname_semicolon_first…
#>  3 "Baeza, J., "           fix_author_name_by_deleting_last_semicolon           
#>  4 "Bodo Weigert"          fix_firstname_lastname_with_lastname_semicolon_first…
#>  5 "Colon, J., "           fix_author_name_by_deleting_last_semicolon           
#>  6 "Fanck, Britta,"        fix_author_name_by_deleting_last_semicolon           
#>  7 "Ferreira, B., "        fix_author_name_by_deleting_last_semicolon           
#>  8 "Fischer, Marcel,"      fix_author_name_by_deleting_last_semicolon           
#>  9 "Frison, N., "          fix_author_name_by_deleting_last_semicolon           
#> 10 "Fritz, Birgit,"        fix_author_name_by_deleting_last_semicolon           
#> 11 "Gorostegi, N., "       fix_author_name_by_deleting_last_semicolon           
#> 12 "Holzbecher, Ekkehard," fix_author_name_by_deleting_last_semicolon           
#> 13 "Johan Stüber"          fix_firstname_lastname_with_lastname_semicolon_first…
#> 14 "Jucnic-Zonta, Z., "    fix_author_name_by_deleting_last_semicolon           
#> 15 "Lahaye, C., "          fix_author_name_by_deleting_last_semicolon           
#> 16 "Misiukas, J., "        fix_author_name_by_deleting_last_semicolon           
#> 17 "Pascale Rouault"       fix_firstname_lastname_with_lastname_semicolon_first…
#> 18 "Paul Uwe Thamsen"      fix_firstname_lastname_with_lastname_semicolon_first…
#> 19 "Ponsa, S., "           fix_author_name_by_deleting_last_semicolon