setwd("~/tools/dcupload")
sink("metadata.txt")
cat("
# This file, referred to as a collection parameter file, consists of
# data in key=value pairs. This data is sent to the ANU Data Commons
# to create a collection, establish relations with other records,
# and/or upload files to those collections.
 
# The metadata section consists of metadata for use in creation (not
# for modification) of record metadata in ANU Data Commons. The
# following fields are required for the creation of a record. The file
# Keys.txt contains the full list of values that can be specified in
# this file. Based on this information below, a collection record of
# type databaset with the title "Test Collection 6/05/2013" will be
# created owned by Meteorology and Health group.
[metadata]
type = Collection
subType = dataset
ownerGroup = 5
# 6 on production, 5 on dev
name = Civil Status, Gender and Activity Sector
briefDesc = An example, fictional dataset for Decision Tree Models
citationCreator = Ritschard, G. (2006). Computing and using the deviance with classification trees. In Compstat 2006 - Proceedings in Computational Statistics 17th Symposium Held in Rome, Italy, 2006.
email = ivan.hanigan@anu.edu.au
anzforSubject = 1601
 
# The relations section allows you to specify the relation this record
# has with other records in the system.  Currently relations with NLA
# identifiers is not supported.
[relations]
isOutputOf = anudc:123
 
# This section contains a line of the form 'pid = anudc:123' once a
# record has been created so executing the uploader script with the
# same collection parameter file doesnt create a new record with the
# same metadata.
[pid]
")
sink()

# run the dcload
system("python3 dcuploader.py -c metadata.txt")

# func
require(devtools)
install_github("reml", "ropensci")
require(reml)
install_github("rfigshare", "ropensci")
require(rfigshare)
install_github("disentangle", "ivanhanigan")
require(disentangle)
# load
fpath <- system.file(file.path("extdata","civst_gend_sector_eml.xml"), package = "disentangle")
setwd(dirname(fpath))
obj <- eml_read(fpath)
# clean
obj
# do

## STEP 1: find one of the preset categories
# available. We can ask the API for
# a list of all the categories:
list <- fs_category_list()
list[grep("Survey", list)]

## STEP 2: PUBLISH TO FIGSHARE
id <- eml_publish(fname,
                  description="Example EML
                    A fictional dataset",
                  categories = "Survey results",
                  tags = "EML",
                  destination="figshare"
                  )
# there are several warnings
# but go to figshare and it has sent the metadata and data OK

# make public using either the figshare web interface, the
# rfigshare package (using fs_make_public(id)) or just by adding
# the argument visibility = TRUE to the above eml_publish
fs_make_public(id)

# func
require(devtools)
install_github("reml", "ropensci")
require(reml)
?eml_write

install_github("disentangle", "ivanhanigan") # for the data
                                             # described in prev post

# load
fpath <- system.file(file.path("extdata", "civst_gend_sector.csv"),
                     package = "disentangle"
                     )
civst_gend_sector <- read.csv(fpath)

# clean
str(civst_gend_sector)

# do
eml_write(civst_gend_sector,
          creator = "Ivan Hanigan <ivanhanigan@gmail.com>")


          


# Starts up the wizard, a section is shown below.  The wizard
# prompts in the console and the user writes the answer.

# Enter description for column 'civil_status':
#  marriage status
# column civil_status appears to contain categorical data.
#  
# Categories are divorced/widowed, married, single
#  Please define each of the categories at the prompt
# define 'divorced/widowed':
# was once married
# define 'married':
# still married
# define 'single':
# never married

# TODO I don't really know what activity_sector is.  I assumed
# school because Categories are primary, secondary, tertiary.

# this created "metadata.xml" and "metadata.csv"
file.remove(c("metadata.xml","metadata.csv"))

ds <- data.set(civst_gend_sector,
               col.defs = c("Marriage status", "sex", "education", "counts"),
               unit.defs = list(c("was once married","still married","never married"),
                   c("women", "men"),
                   c("primary school","secondary school","tertiary school"),
                   c("persons"))
               )
ds
# this prints the dataset and the metadata
# now run the EML function
eml_write(ds, 
          title = "civst_gend_sector",  
          description = "An example, fictional dataset for Decision Tree Models",
          creator = "Ivan Hanigan <ivanhanigan@gmail.com>",
          file = "inst/extdata/civst_gend_sector_eml.xml"
          )
# this created the xml and csv with out asking anything
# but returned a
## Warning message:
## In `[<-.data.frame`(`*tmp*`, , value = list(civil_status = c(2L,  :
##   Setting class(x) to NULL;   result will no longer be an S4 object

# TODO investigate this?

# now we can access the local EML
obj <- eml_read("inst/extdata/civst_gend_sector_eml.xml")
obj 
str(dataTable(obj))
# returns an error
## Error in plyr::compact(lapply(slotNames(from), function(s) if (!isEmpty(slot(from,  (from attribute.R#300) : 
##   subscript out of bounds

# Filter rows by criteria
subset(airquality, Temp > 90, select = c(Ozone, Temp))

## NB This is a convenience function intended for use interactively.  For
## programming it is better to use the standard subsetting functions like
## ‘[’, and in particular the non-standard evaluation of argument
## ‘subset’ can have unanticipated consequences.

with(airquality,
     airquality[Temp > 90, c("Ozone", "Temp")]
     )

# OR

airquality[airquality$Temp > 90,  c("Ozone", "Temp")] #### R-transform
# New columns that are functions of other columns       
df <- transform(airquality,
                new = -Ozone,
                Temp2 = (Temp-32)/1.8
                )
head(df) #### R-mutate
require(plyr)
# same thing as transform
df <- mutate(airquality, new = -Ozone, Temp = (Temp - 32) / 1.8)    
# Things transform can't do
df <- mutate(airquality, Temp = (Temp - 32) / 1.8, OzT = Ozone / Temp)

# mutate is rather faster than transform
system.time(transform(baseball, avg_ab = ab / g))
system.time(mutate(baseball, avg_ab = ab / g)) #### R-summarise
# New data.frame where columns are functions of existing columns
require(plyr)    
df <- ddply(.data = airquality,
            .variables = "Month",
            .fun = summarise,
            tmax = max(Temp),
            tav = mean(Temp),
            ndays = length(unique(Day))
            )
head(df)

# Notice how the name of the variable Temp doesn't need quotes?
# this means that you need to hard code the names
# But if you want to pass variables to this inside a function we need a
# different approach.

summarise_df  <- function(x, by, var1, var2, var3)
  {
    data_out <- ddply(x,
                      by,
                      function(df) return(
                        c(
                          tmax = max(df[,var1]),
                          tav = mean(df[,var2]),
                          ndays = length(unique(df[,var3]))
                          )
                        )
                      )
    return(data_out)
  }

df2 <- summarise_df(x = airquality, by = "Month",
                   var1 = "Temp", var2 = "Temp", var3 = "Day"
                   )

head(df2)
all.equal(df,df2)
# TRUE

summarise_df2  <- function(x, by, var1, var2, var3)
  {
    data_out <- eval(
      parse(
        text =
        sprintf(
          "ddply(.data = %s,
            .variables = '%s',
            .fun = summarise,
            tmax = max(%s),
            tav = mean(%s),
            ndays = length(unique(%s))
            )", x, by, var1, var2, var3
          )
        )
      )
    return(data_out)
  }

df3 <- summarise_df2(x = "airquality", by = "Month",
                     var1 = "Temp", var2 = "Temp", var3 = "Day"
                     )
head(df3)
all.equal(df, df3)
# TRUE #### R-arrange
# Re-order the rows of a data.frame
df <- arrange(airquality, Temp, Ozone)
head(df)