###############################################################
## Pretest Description
## These functions are designed to build the pretest description
## function from the AssessmentModelDescription
## Resets column counter
## Is there a better way than setting this as a global variable?
"resetPDColumns" <-
function(zeroColumn=0) {
pdColumnCount <<-zeroColumn
}
## Increments counter and returns the next value.
"nextPDColumn" <-
function () {
pdColumnCount <<- pdColumnCount+1
pdColumnCount
}
## This produces a set of keys to use in the generation of
## a translation table. The behavior depends on the value of the
## intCode argument
## intCode == NULL -- key=value (identity map)
## intCode == 1 -- 1 based coding, lowest (last) to highest (first)
## intCode == 0 -- 0 based coding, lowest (last) to highest (first)
## intCode == -1 -- StatShop 0 based coding, highest(first) to lowest(last)
"integerCodes" <-
function(values,intCode) {
if (is.null(intCode)) {
key <- values
} else {
key <- length(values):1
if (intCode==0) {
key <- key -1
}
if (intCode < 0) {
key <- rev(key-1)
}
}
names(key) <- values
key
}
## Outputs a translation table in XML format
"writeTranslationTable" <-
function(pdf,keys,values,indent) {
cat(indent,"\n",sep="",file=pdf)
# print(values)
# print(keys)
for (i in 1:length(keys)) {
cat(indent," ",
"",keys[i],"",
"",values[i],"",
"\n",sep="",file=pdf)
}
cat(indent,"\n",sep="",file=pdf)
}
## Read a translation table in XML format and returns a list in
## key = value format.
"readTranslationTable" <-
function(xmlNode) {
kvp <- xmlElementsByTagName(xmlNode,"KeyValuePair")
keys <- sapply(kvp,function(pair) {xmlValue(pair[["Key"]])})
values <- sapply(kvp,function(pair) {xmlValue(pair[["Value"]])})
names(values) <- keys
as.list(values)
}
## Read a translation table in XML format and returns a list in
## key = value format.
"readRevTranslationTable" <-
function(xmlNode) {
kvp <- xmlElementsByTagName(xmlNode,"KeyValuePair")
keys <- sapply(kvp,function(pair) {xmlValue(pair[["Key"]])})
values <- sapply(kvp,function(pair) {xmlValue(pair[["Value"]])})
names(keys) <- values
as.list(keys)
}
## Some common values for NA codes
omitCode789 <- c("NA"=7,"Omit"=8,"Not Reached"=9)
omitCodeStrings <- c("NA"="NA","Omit"="Omit","Not Reached"="NR")
omitCodeGENASYS <- c("NA"="X","Omit"="M","Not Reached"="N")
### This function reads a CSV file relative to a layout and builds a
### data frames corresponding to the demographic variables, statistics
### and task responses.
"read.cpd.csv" <-
function(csvfile, layoutfile,hasUIDs=TRUE) {
## TODO accomodate headers
rawdata <- read.csv(csvfile,header=FALSE,row.names=ifelse(hasUIDs,1,FALSE))
## Now Parse Pretest Description File.
pd <- xmlRoot(xmlTreeParse(layoutfile))
if (!hasUIDs) {
stem <- xmlValue(pd[["UserIDPrefix"]])
row.names(rawdata) <- paste(stem,0:nrow(rawdata)-1,sep="")
}
resetPDColumns()
demometa <- pd[["DemographicVariables"]]
if (xmlSize(demometa) > 0) {
demographics <- readDemographicCSVData(rawdata,demometa)
} else {
demographics <- NA
}
statmeta <- pd[["Statistics"]]
if (xmlSize(statmeta) >0 ) {
statistics <- readStatisticCSVData(rawdata,statmeta)
} else {
statistics <- NA
}
taskNodes <- xmlElementsByTagName(pd[["Tasks"]],"Task")
if (length(taskNodes) > 0) {
taskIDs <- sapply(taskNodes,function(node) {
xmlValue(node[["TaskID"]])
})
## Last element of the task is a useful task name.
tnames <- sapply(strsplit(taskIDs,"/"), function(l) {rev(l)[1]})
names(taskIDs)<-tnames
taskData <- lapply(taskNodes, readTaskCSVData, rawdata)
names(taskData) <- tnames
} else {
taskIDs <- character()
taskData <- list()
}
list(demographics=demographics,statistics=statistics,taskIDs=taskIDs,taskData=taskData)
}
## Returns the demographic variables
"readDemographicCSVData" <-
function(rawdata,xmlNode) {
result <- NULL
for (dmNode in xmlElementsByTagName(xmlNode,"DemographicVariableMeta")) {
## Column order is normative, not the column attribute.
index <- nextPDColumn()
name <- xmlGetAttr(dmNode,"name")
column <- rawdata[,index]
## recode/reorder factor variable.
ttnode <- xmlElementsByTagName(dmNode,"TranslationTable")
if (length(ttnode) > 0) {
tt <- readRevTranslationTable(ttnode[[1]])
if (is.numeric(column)) {
column <- as.factor(column)
levels(column) <- lapply(tt,as.numeric)
} else {
levels(column) <- tt
}
}
result <- addCol(result,column,name)
}
row.names(result) <- row.names(rawdata)
result
}
## Bind column to result
addCol <-
function(result, column,name) {
rnames <- row.names(result)
if (is.null(result) || ncol(result) == 0) {
result <- data.frame(column)
names(result) <- name
rnames <- names(column)
} else {
index <- ncol(result)+1
result <- data.frame(result,column)
names(result)[index] <- name
}
if (! is.null(rnames)) {
row.names(result) <- rnames
}
result
}
## Returns the statistics
"readStatisticCSVData" <-
function(rawdata,xmlNode) {
result <- NULL
varNodes <-
for (dmNode in xmlElementsByTagName(xmlNode,"StatisticMeta")) {
type <- xmlGetAttr(dmNode,"type") # For some reason need "type" not "xsi:type"
type <- sub("^.*:","",type)
name <- xmlGetAttr(dmNode,"name")
if (type == "ProbabilityStatisticMetaType") {
lnodes <- xmlElementsByTagName(dmNode,"Level")
levels <- sapply(lnodes, xmlGetAttr,"name")
## burn up the correct number of columns
indexes <- sapply(lnodes, function(n) nextPDColumn())
columns <- rawdata[,indexes]
names(columns) <- paste(name,levels,sep=":")
if (is.null(result)) {
result <- columns
} else {
result <- data.frame(result,columns)
}
} else {
## Default Processing should work fine here.
## Column order is normative, not the column attribute.
index <- nextPDColumn()
column <- rawdata[,index]
## recode/reorder factor variable.
ttnode <- xmlElementsByTagName(dmNode,"TranslationTable")
if (length(ttnode) > 0) {
tt <- readRevTranslationTable(ttnode[[1]])
if (is.numeric(column)) {
column <- as.factor(column)
levels(column) <- lapply(tt,as.numeric)
} else {
levels(column) <- tt
}
}
result <- addCol(result,column,name)
}
}
row.names(result) <- row.names(rawdata)
result
}
## Returns the Observable values
"readTaskCSVData" <-
function(xmlNode,rawdata) {
result <- NULL
for (obsNode in xmlElementsByTagName(xmlNode,"Observation")) {
## Column order is normative, not the column attribute.
index <- nextPDColumn()
name <- xmlGetAttr(obsNode,"name")
column <- rawdata[,index]
## recode/reorder factor variable.
ttnode <- xmlElementsByTagName(obsNode,"TranslationTable")
if (length(ttnode) > 0) {
tt <- readRevTranslationTable(ttnode[[1]])
if (is.numeric(column)) {
column <- as.factor(column)
levels(column) <- lapply(tt,as.numeric)
} else {
levels(column) <- tt
}
}
result <- addCol(result,column,name)
}
row.names(result) <- row.names(rawdata)
result
}
###################################################
## Statistic Meta Objects
## column is the column where the data is stored. -1 is a special
## value saying the next available column should be used.
## columnID is the value for the name.
##In the case
## Type should be one of c("Integer","Real","Discrete","Probability")
StatisticMetaTypes <- c("Integer","Real","Discrete","Probability")
## In the case of "Probability" statistics, column is interpreted as
## the location of the first column and columnID is interpreted as a
## base name to which the level name will be appended.
"StatisticMeta" <-
function (name, type, column=-1, columnID=name, translationTable=NULL,
levels=character())
{
if (!(type %in% StatisticMetaTypes))
stop("Unrecognized statistic meta type", type,
"for statistic",name)
if (length(levels)>0 && type != "Probability")
stop("Levels only supported for Probability statistics, name = ",
name)
stat <- list(name = name, type=type, column=column,
columnID=columnID, translationTable=translationTable,
levels=levels)
class(stat) <- "StatisticMeta"
stat
}
"print.StatisticMeta" <-
function (x, ...)
{
print(paste("", sep=""), ...)
}
"toString.StatisticMeta" <-
function (x, ...)
{
paste("")
}
StatisticToMetaMap <- c("Bayes Net Mean"="Real",
"Bayes Net Standard Deviation"="Real",
"Bayes Net Quantile"="Discrete",
"Bayes Net Mode"="Discrete",
"Bayes Net Margin"="Probability",
"Sum at Level"="Real",
"Probability of Threshold"="Real",
"Probability of Configuration"="Real")
"StatisticToMeta" <-
function (stat, vars, codeRule=NULL) {
name <- stat$name
type <- StatisticToMetaMap[stat$class]
if (is.na(type)) {
stop("Can't find statistic type for class ", stat$class,
", Statistic = ",toString(stat))
}
columnID <- sub("\\(",".",(sub("\\)",".",name)))
##Default values
translationTable <- NULL
levels <- character()
if (type == "Discrete") {
var <- vars[[stat$reportingVars]]
translationTable <- integerCodes(var$levels,codeRule)
}
if (type == "Probability") {
var <- vars[[stat$reportingVars]]
levels <- var$levels
columnID <- paste(columnID,".",sep="")
}
StatisticMeta(name,type, -1, columnID, translationTable, levels)
}
xmlStatisticMeta <- function (stat, nextCol = quote(nextPDColumn())) {
attr <- c(name=stat$name,
"xsi:type"=paste(pd.prefix,":",
stat$type,"StatisticMetaType",sep=""))
if (length(stat$levels) == 0) {
attr <- c(attr, c(column=as.character(eval(nextCol)),
columnID=stat$columnID))
}
statNode <- xmlNode("StatisticMeta",attrs=attr)
if (!is.null(stat$translationTable)) {
statNode <- xmlAddChild(statNode,
xmlTranslationTable(stat$translationTable,
names(stat$translationTable)))
}
if (length(stat$levels)>0) {
for (state in stat$levels) {
lat <- c(name=state, column=as.character(eval(nextCol)),
columnID=paste(stat$columnID,state,sep=""))
statNode <- xmlAddChild(statNode, xmlNode("Level",attrs = lat))
}
}
statNode
}
"parseStatisticMeta" <-
function (statNode) {
name <- xmlGetAttr(statNode, "name")
column <- xmlGetAttr(statNode, "column", -1)
columnID <- xmlGetAttr(statNode, "columnID", NULL)
xmltype <- xmlGetAttr(statNode, "type")
type <- sub("StatisticMetaType","",sub("^.*:","",xmltype))
if (is.null(columnID))
columnID <- ifelse(type=="Probability",paste(name,".",sep=""),name)
translationTable <- NULL
if (!is.null(statNode[["TranslationTable"]])) {
translationTable <-
readRevTranslationTable(statNode[["TranslationTable"]])
}
levels <- character()
levNodes <- xmlElementsByTagName(statNode,"Level")
if (length(levNodes) >0) {
levels <- sapply(levNodes,function (n) xmlGetAttr(n,"name"))
names(levels) <- NULL
## Certain values are taken from the first Level element
lev1 <- levNodes[[1]]
column <- xmlGetAttr(lev1,"column",column)
columnID <- xmlGetAttr(lev1,"columnID",columnID)
columnID <- sub(levels[1],"",columnID)
}
StatisticMeta(name,type,column,columnID,translationTable,levels)
}
##############################################################
## DemographicVariableMeta
DemographicMetaTypes <- c("Integer","Real","Discrete","String")
"DemographicVariableMeta" <-
function (name, type, column=-1, columnID=name, translationTable=NULL)
{
if (!(type %in% DemographicMetaTypes))
stop("Unrecognized Demograpic Variable type", type,
"for statistic",name)
dem <- list(name = name, type=type, column=column,
columnID=columnID, translationTable=translationTable)
class(dem) <- "DemographicVariableMeta"
dem
}
"print.DemographicVariableMeta" <-
function (x, ...)
{
print(paste("", sep=""), ...)
}
"toString.DemographicVariableMeta" <-
function (x, ...)
{
paste("")
}
buildDemographicMetaList <- function (list, codeRule=NULL) {
lapply(names(list), function(name) {
tt <- NULL
type <- list[[name]]
if (length(type) == 1) {
if (type == "Discrete") {
warning("Unexpected literal type 'Discrete', expected list of levels.")
}
} else {
tt <- integerCodes(type,codeRule)
type <- "Discrete"
}
DemographicVariableMeta(name,type,translationTable=tt)
})
}
xmlDemographicVariableMeta <- function (dem, column=nextPDColumn()) {
attr <- c(name=dem$name,
"xsi:type"=paste(pd.prefix,":",
dem$type,"DemographicVariableMetaType",sep=""),
column=as.character(column),
columnID=dem$columnID)
demNode <- xmlNode("DemographicVariableMeta",attrs=attr)
if (!is.null(dem$translationTable)) {
demNode <- xmlAddChild(demNode,
xmlTranslationTable(dem$translationTable,
names(dem$translationTable)))
}
demNode
}
"parseDemographicVariableMeta" <-
function (demNode) {
name <- xmlGetAttr(demNode, "name")
column <- xmlGetAttr(demNode, "column", -1)
columnID <- xmlGetAttr(demNode, "columnID", NULL)
if (is.null(columnID)) columnID <- name
xmltype <- xmlGetAttr(demNode, "type")
type <- sub("DemographicVariableMetaType","",sub("^.*:","",xmltype))
translationTable <- NULL
if (!is.null(demNode[["TranslationTable"]])) {
translationTable <-
readRevTranslationTable(demNode[["TranslationTable"]])
}
DemographicVariableMeta(name,type,column,columnID,translationTable)
}
############################################################
## ObservationMeta
"ObservationMeta" <-
function (name, column=-1, columnID=name, translationTable=NULL)
{
obs <- list(name = name, column=column,
columnID=columnID,
translationTable=translationTable)
class(obs) <- "ObservationMeta"
obs
}
"print.ObservationMeta" <-
function (x, ...)
{
print(paste(""), ...)
}
"toString.ObservationMeta" <-
function (x, ...)
{
paste("")
}
xmlObservationMeta <- function (obs, column=nextPDColumn()) {
attr <- c(name=obs$name,
column=as.character(column),
columnID=obs$columnID)
obsNode <- xmlNode("Observation",attrs=attr)
if (!is.null(obs$translationTable)) {
obsNode <- xmlAddChild(obsNode,
xmlTranslationTable(obs$translationTable,
names(obs$translationTable)))
}
obsNode
}
"parseObservationMeta" <-
function (obsNode) {
name <- xmlGetAttr(obsNode, "name")
column <- xmlGetAttr(obsNode, "column", -1)
columnID <- xmlGetAttr(obsNode, "columnID", NULL)
if (is.null(columnID)) columnID <- name
translationTable <- NULL
if (!is.null(obsNode[["TranslationTable"]])) {
translationTable <-
readRevTranslationTable(obsNode[["TranslationTable"]])
}
ObservationMeta(name,column,columnID,translationTable)
}
#############################################################
## TaskMeta (the actual element is called, task, but it is really a Meta element).
"TaskMeta" <-
function (taskID, observations)
{
tm <- list(taskID = taskID, observations=observations)
class(tm) <- "TaskMeta"
tm
}
"print.TaskMeta" <-
function (x, ...)
{
print(paste(""), ...)
}
"toString.TaskMeta" <-
function (x, ...)
{
paste("")
}
xmlTaskMeta <- function (tm, nextCol=quote(nextPDColumn())) {
tmNode <- xmlNode("Task")
tmNode <- xmlAddChild(tmNode,xmlNode("TaskID",tm$taskID))
for (obs in tm$observations) {
tmNode <- xmlAddChild(tmNode,
xmlObservationMeta(obs, eval(nextCol)))
}
tmNode
}
"parseTaskMeta" <-
function (tmNode) {
taskID <- xmlValue(tmNode[["TaskID"]])
observations <- lapply(xmlElementsByTagName(tmNode,"Observation"),
parseObservationMeta)
names(observations) <- sapply(observations, function(obs)
obs$name)
TaskMeta(taskID,observations)
}
##############################################
## PretestDescripton
"PretestDescription" <-
function(appID,uidPrefix,missingHandlers=list(),
demographics=list(), statistics=list(),
tasks=list()) {
pd <- list(appID=appID, uidPrefix=uidPrefix, missingHandlers=missingHandlers,
demographics=demographics, statistics=statistics, tasks=tasks)
class(pd) <- "PretestDescription"
pd
}
xmlPretestDescription <-
function (pd, toplevel=TRUE) {
pdNode <- xmlNode("PretestDescription")
if (toplevel) {
attrs <- c("xmlns:xsi"=xmlns.xsi,
"xsi:schemaLocation"=paste(pd.namespace,pd.schemaLocation,
sep=" "),
"xmlns"=pd.namespace)
names(attrs)[3] <- paste("xmlns",pd.prefix,sep=":")
pdNode <- xmlNode("PretestDescription", attrs=attrs,
namespace=pd.prefix)
}
pdNode <- xmlAddChild(pdNode, xmlNode("AppID", pd$appID))
pdNode <- xmlAddChild(pdNode, xmlNode("UserIDPrefix", pd$uidPrefix))
## Missing Codes
pdNode <- append.XMLNode(pdNode, lapply(pd$missingHandlers,xmlMissingCode))
resetPDColumns()
nextCol <- quote(nextPDColumn())
## Demographic Variables
demoNode <- xmlNode("DemographicVariables")
for (dem in pd$demographics) {
demoNode <- xmlAddChild(demoNode,
xmlDemographicVariableMeta(dem,eval(nextCol)))
}
pdNode <- xmlAddChild(pdNode, demoNode)
## Statistics
statNode <- xmlNode("Statistics")
for (stat in pd$statistics) {
statNode <- xmlAddChild(statNode,
xmlStatisticMeta(stat,nextCol))
}
pdNode <- xmlAddChild(pdNode, statNode)
tasksNode <- xmlNode("Tasks")
for (task in pd$tasks) {
tasksNode <- xmlAddChild(tasksNode,
xmlTaskMeta(task,nextCol))
}
pdNode <- xmlAddChild(pdNode,tasksNode)
pdNode
}
"parsePretestDescription" <-
function(xmlNode) {
appID <- xmlValue(xmlNode[["AppID"]])
uidPrefix <- xmlValue(xmlNode[["uidPrefix"]])
missingHandlers <- lapply(xmlElementByTagName(xmlNode,"MissingCode"),
parseMissingCode)
names(missingHandlers) <- sapply(missingHandlers, function (m) m$type)
demographics <- lapply(xmlChildren(xmlNode[["DemographicVariables"]]),
parseDemographicVariableMeta)
names(demographics) <- sapply(demographics, function (d) d$name)
statistics <- lapply(xmlChildren(xmlNode[["Statistics"]]),
parseStatisticMeta)
names(statistics) <- sapply(statistics, function (s) s$name)
tasks <- lapply(xmlChildren(xmlNode[["Tasks"]]), parseTaskMeta)
names(tasks) <- sapply(tasks, function (t) getECDIDtask(t$taskID))
PrestestDescription(appID,uidPrefix,missingHandlers,
demographics, statistics, tasks)
}
selectTasksByID <- function (pd,tidlist) {
tids <- sapply(pd$tasks, function(t) t$taskID)
selected <- match(tidlist,tids)
pd$tasks[selected]
}
#########################################################
## Translation Functions
QtoPD <-
function (Qmat, caf, observables, itemColPrefix="Item",
integerCode=NULL,
appID=deparse(substitute(Qmat)), uidPrefix="",
missingHandlers=list(),
demographics=list(),
statistics=list()){
tasks <- list()
taskName <- ""
taskMeta <- NULL
for (irow in 1:nrow(Qmat)) {
if (Qmat[irow,"TaskName"] != taskName) {
## New Task
if (!is.null(taskMeta)) {
tasks <- c(tasks,list(taskMeta))
}
taskName <- Qmat[irow,"TaskName"]
taskModel <- Qmat[irow,"TaskModel"]
taskID <- makeECDID(caf,taskModel,taskName)
taskMeta <- TaskMeta(taskID,list())
}
## Observation specific processing
obsname <- as.character(Qmat[irow,"ObsName"])
if (is.null(observables[[obsname]])) {
stop("Cannot find observable ",obsname," for task ",taskID)
}
values <- observables[[obsname]]$levels
colID <- paste(itemColPrefix,
ifelse(is.null(Qmat$ItemNumber),irow,
Qmat[irow,"ItemNumber"]),
sep="")
obs <- ObservationMeta(obsname, columnID=colID,
translationTable = integerCodes(values,integerCode))
taskMeta$observations <- c(taskMeta$observations,list(obs))
}
## Add last task
if (!is.null(taskMeta)) {
tasks <- c(tasks,list(taskMeta))
}
PretestDescription(appID, uidPrefix, missingHandlers, demographics,
statistics, tasks)
}
## This gets all of the parameters for an Assessment Model Description
## File and puts them into one big group.
## Note, this must run on an Assessment Description, with proper link
## models in order to get the proper task IDs.
## See integerCodes function for description of the behavior of the
## integerCode argument
"amdToPD" <-
function (amd, reportTrueValues=FALSE,
reportStatistics=TRUE,
reportObservables=TRUE,
codeRule=NULL,
omitCodes=c(),
appID = deparse(amd),
uidPrefix = "",
demographics = list()
){
missingHandlers <- processOmitCodes(amd$missingCodes,omitCodes)
smvars <- list()
for (sm in amd$studentModels) {
smvars <- c(smvars,sm$variables)
if (reportTrueValues) {
demographics <- c(demographics,
reportVariablesAsDemographics(sm,codeRule))
}
}
statistics <- list()
if (reportStatistics) {
statistics <-
c(statistics,
lapply(amd$statistics,
function (stat) StatisticToMeta(stat,smvars,codeRule)))
}
tasks <- list()
if (reportObservables) {
taskIDs <- unique(sapply(amd$linkModels,function(lm) lm$taskID))
tasks <- lapply(taskIDs, function (tid) {
reportTasks(tid,amd$linkModels,codeRule)
})
}
PretestDescription(appID,uidPrefix,missingHandlers,
demographics,statistics,tasks)
}
## Write out missing code handers.
"processOmitCodes" <-
function(handlers,omitCodes) {
result <- list()
for (handler in handlers) {
type <- handler$type
code <- omitCodes[type]
if (!is.na(code)) {
result <- c(result,
list(MissingCode(type,"Recode",handler$xmlCode,
handler$checksum,code)))
}
}
}
reportVariablesAsDemographics <-
function(sm, codeRule = NULL) {
mod <- getECDIDModel(sm$id)
lapply(sm$variables, function (var) {
DemographicVariableMeta(paste(mod,var$name,sep="."), "Discrete",
translationTable=integerCodes(var$levels,
codeRule))
})
}
## Write out the tags corresponding to the tasks
"reportTasks" <-
function(tid, lmlist,codeRule) {
obslist <- list()
for (lm in lmlist) {
if (lm$taskID == tid) {
##LM for current task
for (var in lm$variables) {
if (var$role == "Observable") {
obslist <-
c(obslist,
list(ObservationMeta(var$name,
translationTable=integerCodes(var$levels, codeRule))))
}
}
}
}
names(obslist) <- sapply(obslist, function(o) o$name)
TaskMeta(tid,obslist)
}
"reportTaskEM" <-
function(leafid,em,codeRule) {
tid <- paste(em$taskID,leafid,".1",sep="")
obslist <- list()
for (var in em$variables) {
if (var$role == "Observable") {
obslist <-
c(obslist,
list(ObservationMeta(var$name,
translationTable=integerCodes(var$levels, codeRule))))
}
}
names(obslist) <- sapply(obslist, function(o) o$name)
TaskMeta(tid,obslist)
}
buildTasksforQmat <-
function(Q,amd, codeRule=NULL) {
Tasks <- list()
for (i in 1:nrow(Q)) {
Tasks <- c(Tasks,
list(reportTaskEM(Q[i,"TaskName"],
amd$evidence[[Q[i,"EvidenceModel"]]],
codeRule)))
}
names(Tasks) <- Q[,"TaskName"]
Tasks
}