############################################################### ## Pretest Description ## These functions are designed to build the pretest description ## function from the AssessmentModelDescription ## Resets column counter ## Is there a better way than setting this as a global variable? "resetPDColumns" <- function(zeroColumn=0) { pdColumnCount <<-zeroColumn } ## Increments counter and returns the next value. "nextPDColumn" <- function () { pdColumnCount <<- pdColumnCount+1 pdColumnCount } ## This produces a set of keys to use in the generation of ## a translation table. The behavior depends on the value of the ## intCode argument ## intCode == NULL -- key=value (identity map) ## intCode == 1 -- 1 based coding, lowest (last) to highest (first) ## intCode == 0 -- 0 based coding, lowest (last) to highest (first) ## intCode == -1 -- StatShop 0 based coding, highest(first) to lowest(last) "integerCodes" <- function(values,intCode) { if (is.null(intCode)) { key <- values } else { key <- length(values):1 if (intCode==0) { key <- key -1 } if (intCode < 0) { key <- rev(key-1) } } names(key) <- values key } ## Outputs a translation table in XML format "writeTranslationTable" <- function(pdf,keys,values,indent) { cat(indent,"\n",sep="",file=pdf) # print(values) # print(keys) for (i in 1:length(keys)) { cat(indent," ", "",keys[i],"", "",values[i],"", "\n",sep="",file=pdf) } cat(indent,"\n",sep="",file=pdf) } ## Read a translation table in XML format and returns a list in ## key = value format. "readTranslationTable" <- function(xmlNode) { kvp <- xmlElementsByTagName(xmlNode,"KeyValuePair") keys <- sapply(kvp,function(pair) {xmlValue(pair[["Key"]])}) values <- sapply(kvp,function(pair) {xmlValue(pair[["Value"]])}) names(values) <- keys as.list(values) } ## Read a translation table in XML format and returns a list in ## key = value format. "readRevTranslationTable" <- function(xmlNode) { kvp <- xmlElementsByTagName(xmlNode,"KeyValuePair") keys <- sapply(kvp,function(pair) {xmlValue(pair[["Key"]])}) values <- sapply(kvp,function(pair) {xmlValue(pair[["Value"]])}) names(keys) <- values as.list(keys) } ## Some common values for NA codes omitCode789 <- c("NA"=7,"Omit"=8,"Not Reached"=9) omitCodeStrings <- c("NA"="NA","Omit"="Omit","Not Reached"="NR") omitCodeGENASYS <- c("NA"="X","Omit"="M","Not Reached"="N") ### This function reads a CSV file relative to a layout and builds a ### data frames corresponding to the demographic variables, statistics ### and task responses. "read.cpd.csv" <- function(csvfile, layoutfile,hasUIDs=TRUE) { ## TODO accomodate headers rawdata <- read.csv(csvfile,header=FALSE,row.names=ifelse(hasUIDs,1,FALSE)) ## Now Parse Pretest Description File. pd <- xmlRoot(xmlTreeParse(layoutfile)) if (!hasUIDs) { stem <- xmlValue(pd[["UserIDPrefix"]]) row.names(rawdata) <- paste(stem,0:nrow(rawdata)-1,sep="") } resetPDColumns() demometa <- pd[["DemographicVariables"]] if (xmlSize(demometa) > 0) { demographics <- readDemographicCSVData(rawdata,demometa) } else { demographics <- NA } statmeta <- pd[["Statistics"]] if (xmlSize(statmeta) >0 ) { statistics <- readStatisticCSVData(rawdata,statmeta) } else { statistics <- NA } taskNodes <- xmlElementsByTagName(pd[["Tasks"]],"Task") if (length(taskNodes) > 0) { taskIDs <- sapply(taskNodes,function(node) { xmlValue(node[["TaskID"]]) }) ## Last element of the task is a useful task name. tnames <- sapply(strsplit(taskIDs,"/"), function(l) {rev(l)[1]}) names(taskIDs)<-tnames taskData <- lapply(taskNodes, readTaskCSVData, rawdata) names(taskData) <- tnames } else { taskIDs <- character() taskData <- list() } list(demographics=demographics,statistics=statistics,taskIDs=taskIDs,taskData=taskData) } ## Returns the demographic variables "readDemographicCSVData" <- function(rawdata,xmlNode) { result <- NULL for (dmNode in xmlElementsByTagName(xmlNode,"DemographicVariableMeta")) { ## Column order is normative, not the column attribute. index <- nextPDColumn() name <- xmlGetAttr(dmNode,"name") column <- rawdata[,index] ## recode/reorder factor variable. ttnode <- xmlElementsByTagName(dmNode,"TranslationTable") if (length(ttnode) > 0) { tt <- readRevTranslationTable(ttnode[[1]]) if (is.numeric(column)) { column <- as.factor(column) levels(column) <- lapply(tt,as.numeric) } else { levels(column) <- tt } } result <- addCol(result,column,name) } row.names(result) <- row.names(rawdata) result } ## Bind column to result addCol <- function(result, column,name) { rnames <- row.names(result) if (is.null(result) || ncol(result) == 0) { result <- data.frame(column) names(result) <- name rnames <- names(column) } else { index <- ncol(result)+1 result <- data.frame(result,column) names(result)[index] <- name } if (! is.null(rnames)) { row.names(result) <- rnames } result } ## Returns the statistics "readStatisticCSVData" <- function(rawdata,xmlNode) { result <- NULL varNodes <- for (dmNode in xmlElementsByTagName(xmlNode,"StatisticMeta")) { type <- xmlGetAttr(dmNode,"type") # For some reason need "type" not "xsi:type" type <- sub("^.*:","",type) name <- xmlGetAttr(dmNode,"name") if (type == "ProbabilityStatisticMetaType") { lnodes <- xmlElementsByTagName(dmNode,"Level") levels <- sapply(lnodes, xmlGetAttr,"name") ## burn up the correct number of columns indexes <- sapply(lnodes, function(n) nextPDColumn()) columns <- rawdata[,indexes] names(columns) <- paste(name,levels,sep=":") if (is.null(result)) { result <- columns } else { result <- data.frame(result,columns) } } else { ## Default Processing should work fine here. ## Column order is normative, not the column attribute. index <- nextPDColumn() column <- rawdata[,index] ## recode/reorder factor variable. ttnode <- xmlElementsByTagName(dmNode,"TranslationTable") if (length(ttnode) > 0) { tt <- readRevTranslationTable(ttnode[[1]]) if (is.numeric(column)) { column <- as.factor(column) levels(column) <- lapply(tt,as.numeric) } else { levels(column) <- tt } } result <- addCol(result,column,name) } } row.names(result) <- row.names(rawdata) result } ## Returns the Observable values "readTaskCSVData" <- function(xmlNode,rawdata) { result <- NULL for (obsNode in xmlElementsByTagName(xmlNode,"Observation")) { ## Column order is normative, not the column attribute. index <- nextPDColumn() name <- xmlGetAttr(obsNode,"name") column <- rawdata[,index] ## recode/reorder factor variable. ttnode <- xmlElementsByTagName(obsNode,"TranslationTable") if (length(ttnode) > 0) { tt <- readRevTranslationTable(ttnode[[1]]) if (is.numeric(column)) { column <- as.factor(column) levels(column) <- lapply(tt,as.numeric) } else { levels(column) <- tt } } result <- addCol(result,column,name) } row.names(result) <- row.names(rawdata) result } ################################################### ## Statistic Meta Objects ## column is the column where the data is stored. -1 is a special ## value saying the next available column should be used. ## columnID is the value for the name. ##In the case ## Type should be one of c("Integer","Real","Discrete","Probability") StatisticMetaTypes <- c("Integer","Real","Discrete","Probability") ## In the case of "Probability" statistics, column is interpreted as ## the location of the first column and columnID is interpreted as a ## base name to which the level name will be appended. "StatisticMeta" <- function (name, type, column=-1, columnID=name, translationTable=NULL, levels=character()) { if (!(type %in% StatisticMetaTypes)) stop("Unrecognized statistic meta type", type, "for statistic",name) if (length(levels)>0 && type != "Probability") stop("Levels only supported for Probability statistics, name = ", name) stat <- list(name = name, type=type, column=column, columnID=columnID, translationTable=translationTable, levels=levels) class(stat) <- "StatisticMeta" stat } "print.StatisticMeta" <- function (x, ...) { print(paste("", sep=""), ...) } "toString.StatisticMeta" <- function (x, ...) { paste("") } StatisticToMetaMap <- c("Bayes Net Mean"="Real", "Bayes Net Standard Deviation"="Real", "Bayes Net Quantile"="Discrete", "Bayes Net Mode"="Discrete", "Bayes Net Margin"="Probability", "Sum at Level"="Real", "Probability of Threshold"="Real", "Probability of Configuration"="Real") "StatisticToMeta" <- function (stat, vars, codeRule=NULL) { name <- stat$name type <- StatisticToMetaMap[stat$class] if (is.na(type)) { stop("Can't find statistic type for class ", stat$class, ", Statistic = ",toString(stat)) } columnID <- sub("\\(",".",(sub("\\)",".",name))) ##Default values translationTable <- NULL levels <- character() if (type == "Discrete") { var <- vars[[stat$reportingVars]] translationTable <- integerCodes(var$levels,codeRule) } if (type == "Probability") { var <- vars[[stat$reportingVars]] levels <- var$levels columnID <- paste(columnID,".",sep="") } StatisticMeta(name,type, -1, columnID, translationTable, levels) } xmlStatisticMeta <- function (stat, nextCol = quote(nextPDColumn())) { attr <- c(name=stat$name, "xsi:type"=paste(pd.prefix,":", stat$type,"StatisticMetaType",sep="")) if (length(stat$levels) == 0) { attr <- c(attr, c(column=as.character(eval(nextCol)), columnID=stat$columnID)) } statNode <- xmlNode("StatisticMeta",attrs=attr) if (!is.null(stat$translationTable)) { statNode <- xmlAddChild(statNode, xmlTranslationTable(stat$translationTable, names(stat$translationTable))) } if (length(stat$levels)>0) { for (state in stat$levels) { lat <- c(name=state, column=as.character(eval(nextCol)), columnID=paste(stat$columnID,state,sep="")) statNode <- xmlAddChild(statNode, xmlNode("Level",attrs = lat)) } } statNode } "parseStatisticMeta" <- function (statNode) { name <- xmlGetAttr(statNode, "name") column <- xmlGetAttr(statNode, "column", -1) columnID <- xmlGetAttr(statNode, "columnID", NULL) xmltype <- xmlGetAttr(statNode, "type") type <- sub("StatisticMetaType","",sub("^.*:","",xmltype)) if (is.null(columnID)) columnID <- ifelse(type=="Probability",paste(name,".",sep=""),name) translationTable <- NULL if (!is.null(statNode[["TranslationTable"]])) { translationTable <- readRevTranslationTable(statNode[["TranslationTable"]]) } levels <- character() levNodes <- xmlElementsByTagName(statNode,"Level") if (length(levNodes) >0) { levels <- sapply(levNodes,function (n) xmlGetAttr(n,"name")) names(levels) <- NULL ## Certain values are taken from the first Level element lev1 <- levNodes[[1]] column <- xmlGetAttr(lev1,"column",column) columnID <- xmlGetAttr(lev1,"columnID",columnID) columnID <- sub(levels[1],"",columnID) } StatisticMeta(name,type,column,columnID,translationTable,levels) } ############################################################## ## DemographicVariableMeta DemographicMetaTypes <- c("Integer","Real","Discrete","String") "DemographicVariableMeta" <- function (name, type, column=-1, columnID=name, translationTable=NULL) { if (!(type %in% DemographicMetaTypes)) stop("Unrecognized Demograpic Variable type", type, "for statistic",name) dem <- list(name = name, type=type, column=column, columnID=columnID, translationTable=translationTable) class(dem) <- "DemographicVariableMeta" dem } "print.DemographicVariableMeta" <- function (x, ...) { print(paste("", sep=""), ...) } "toString.DemographicVariableMeta" <- function (x, ...) { paste("") } buildDemographicMetaList <- function (list, codeRule=NULL) { lapply(names(list), function(name) { tt <- NULL type <- list[[name]] if (length(type) == 1) { if (type == "Discrete") { warning("Unexpected literal type 'Discrete', expected list of levels.") } } else { tt <- integerCodes(type,codeRule) type <- "Discrete" } DemographicVariableMeta(name,type,translationTable=tt) }) } xmlDemographicVariableMeta <- function (dem, column=nextPDColumn()) { attr <- c(name=dem$name, "xsi:type"=paste(pd.prefix,":", dem$type,"DemographicVariableMetaType",sep=""), column=as.character(column), columnID=dem$columnID) demNode <- xmlNode("DemographicVariableMeta",attrs=attr) if (!is.null(dem$translationTable)) { demNode <- xmlAddChild(demNode, xmlTranslationTable(dem$translationTable, names(dem$translationTable))) } demNode } "parseDemographicVariableMeta" <- function (demNode) { name <- xmlGetAttr(demNode, "name") column <- xmlGetAttr(demNode, "column", -1) columnID <- xmlGetAttr(demNode, "columnID", NULL) if (is.null(columnID)) columnID <- name xmltype <- xmlGetAttr(demNode, "type") type <- sub("DemographicVariableMetaType","",sub("^.*:","",xmltype)) translationTable <- NULL if (!is.null(demNode[["TranslationTable"]])) { translationTable <- readRevTranslationTable(demNode[["TranslationTable"]]) } DemographicVariableMeta(name,type,column,columnID,translationTable) } ############################################################ ## ObservationMeta "ObservationMeta" <- function (name, column=-1, columnID=name, translationTable=NULL) { obs <- list(name = name, column=column, columnID=columnID, translationTable=translationTable) class(obs) <- "ObservationMeta" obs } "print.ObservationMeta" <- function (x, ...) { print(paste(""), ...) } "toString.ObservationMeta" <- function (x, ...) { paste("") } xmlObservationMeta <- function (obs, column=nextPDColumn()) { attr <- c(name=obs$name, column=as.character(column), columnID=obs$columnID) obsNode <- xmlNode("Observation",attrs=attr) if (!is.null(obs$translationTable)) { obsNode <- xmlAddChild(obsNode, xmlTranslationTable(obs$translationTable, names(obs$translationTable))) } obsNode } "parseObservationMeta" <- function (obsNode) { name <- xmlGetAttr(obsNode, "name") column <- xmlGetAttr(obsNode, "column", -1) columnID <- xmlGetAttr(obsNode, "columnID", NULL) if (is.null(columnID)) columnID <- name translationTable <- NULL if (!is.null(obsNode[["TranslationTable"]])) { translationTable <- readRevTranslationTable(obsNode[["TranslationTable"]]) } ObservationMeta(name,column,columnID,translationTable) } ############################################################# ## TaskMeta (the actual element is called, task, but it is really a Meta element). "TaskMeta" <- function (taskID, observations) { tm <- list(taskID = taskID, observations=observations) class(tm) <- "TaskMeta" tm } "print.TaskMeta" <- function (x, ...) { print(paste(""), ...) } "toString.TaskMeta" <- function (x, ...) { paste("") } xmlTaskMeta <- function (tm, nextCol=quote(nextPDColumn())) { tmNode <- xmlNode("Task") tmNode <- xmlAddChild(tmNode,xmlNode("TaskID",tm$taskID)) for (obs in tm$observations) { tmNode <- xmlAddChild(tmNode, xmlObservationMeta(obs, eval(nextCol))) } tmNode } "parseTaskMeta" <- function (tmNode) { taskID <- xmlValue(tmNode[["TaskID"]]) observations <- lapply(xmlElementsByTagName(tmNode,"Observation"), parseObservationMeta) names(observations) <- sapply(observations, function(obs) obs$name) TaskMeta(taskID,observations) } ############################################## ## PretestDescripton "PretestDescription" <- function(appID,uidPrefix,missingHandlers=list(), demographics=list(), statistics=list(), tasks=list()) { pd <- list(appID=appID, uidPrefix=uidPrefix, missingHandlers=missingHandlers, demographics=demographics, statistics=statistics, tasks=tasks) class(pd) <- "PretestDescription" pd } xmlPretestDescription <- function (pd, toplevel=TRUE) { pdNode <- xmlNode("PretestDescription") if (toplevel) { attrs <- c("xmlns:xsi"=xmlns.xsi, "xsi:schemaLocation"=paste(pd.namespace,pd.schemaLocation, sep=" "), "xmlns"=pd.namespace) names(attrs)[3] <- paste("xmlns",pd.prefix,sep=":") pdNode <- xmlNode("PretestDescription", attrs=attrs, namespace=pd.prefix) } pdNode <- xmlAddChild(pdNode, xmlNode("AppID", pd$appID)) pdNode <- xmlAddChild(pdNode, xmlNode("UserIDPrefix", pd$uidPrefix)) ## Missing Codes pdNode <- append.XMLNode(pdNode, lapply(pd$missingHandlers,xmlMissingCode)) resetPDColumns() nextCol <- quote(nextPDColumn()) ## Demographic Variables demoNode <- xmlNode("DemographicVariables") for (dem in pd$demographics) { demoNode <- xmlAddChild(demoNode, xmlDemographicVariableMeta(dem,eval(nextCol))) } pdNode <- xmlAddChild(pdNode, demoNode) ## Statistics statNode <- xmlNode("Statistics") for (stat in pd$statistics) { statNode <- xmlAddChild(statNode, xmlStatisticMeta(stat,nextCol)) } pdNode <- xmlAddChild(pdNode, statNode) tasksNode <- xmlNode("Tasks") for (task in pd$tasks) { tasksNode <- xmlAddChild(tasksNode, xmlTaskMeta(task,nextCol)) } pdNode <- xmlAddChild(pdNode,tasksNode) pdNode } "parsePretestDescription" <- function(xmlNode) { appID <- xmlValue(xmlNode[["AppID"]]) uidPrefix <- xmlValue(xmlNode[["uidPrefix"]]) missingHandlers <- lapply(xmlElementByTagName(xmlNode,"MissingCode"), parseMissingCode) names(missingHandlers) <- sapply(missingHandlers, function (m) m$type) demographics <- lapply(xmlChildren(xmlNode[["DemographicVariables"]]), parseDemographicVariableMeta) names(demographics) <- sapply(demographics, function (d) d$name) statistics <- lapply(xmlChildren(xmlNode[["Statistics"]]), parseStatisticMeta) names(statistics) <- sapply(statistics, function (s) s$name) tasks <- lapply(xmlChildren(xmlNode[["Tasks"]]), parseTaskMeta) names(tasks) <- sapply(tasks, function (t) getECDIDtask(t$taskID)) PrestestDescription(appID,uidPrefix,missingHandlers, demographics, statistics, tasks) } selectTasksByID <- function (pd,tidlist) { tids <- sapply(pd$tasks, function(t) t$taskID) selected <- match(tidlist,tids) pd$tasks[selected] } ######################################################### ## Translation Functions QtoPD <- function (Qmat, caf, observables, itemColPrefix="Item", integerCode=NULL, appID=deparse(substitute(Qmat)), uidPrefix="", missingHandlers=list(), demographics=list(), statistics=list()){ tasks <- list() taskName <- "" taskMeta <- NULL for (irow in 1:nrow(Qmat)) { if (Qmat[irow,"TaskName"] != taskName) { ## New Task if (!is.null(taskMeta)) { tasks <- c(tasks,list(taskMeta)) } taskName <- Qmat[irow,"TaskName"] taskModel <- Qmat[irow,"TaskModel"] taskID <- makeECDID(caf,taskModel,taskName) taskMeta <- TaskMeta(taskID,list()) } ## Observation specific processing obsname <- as.character(Qmat[irow,"ObsName"]) if (is.null(observables[[obsname]])) { stop("Cannot find observable ",obsname," for task ",taskID) } values <- observables[[obsname]]$levels colID <- paste(itemColPrefix, ifelse(is.null(Qmat$ItemNumber),irow, Qmat[irow,"ItemNumber"]), sep="") obs <- ObservationMeta(obsname, columnID=colID, translationTable = integerCodes(values,integerCode)) taskMeta$observations <- c(taskMeta$observations,list(obs)) } ## Add last task if (!is.null(taskMeta)) { tasks <- c(tasks,list(taskMeta)) } PretestDescription(appID, uidPrefix, missingHandlers, demographics, statistics, tasks) } ## This gets all of the parameters for an Assessment Model Description ## File and puts them into one big group. ## Note, this must run on an Assessment Description, with proper link ## models in order to get the proper task IDs. ## See integerCodes function for description of the behavior of the ## integerCode argument "amdToPD" <- function (amd, reportTrueValues=FALSE, reportStatistics=TRUE, reportObservables=TRUE, codeRule=NULL, omitCodes=c(), appID = deparse(amd), uidPrefix = "", demographics = list() ){ missingHandlers <- processOmitCodes(amd$missingCodes,omitCodes) smvars <- list() for (sm in amd$studentModels) { smvars <- c(smvars,sm$variables) if (reportTrueValues) { demographics <- c(demographics, reportVariablesAsDemographics(sm,codeRule)) } } statistics <- list() if (reportStatistics) { statistics <- c(statistics, lapply(amd$statistics, function (stat) StatisticToMeta(stat,smvars,codeRule))) } tasks <- list() if (reportObservables) { taskIDs <- unique(sapply(amd$linkModels,function(lm) lm$taskID)) tasks <- lapply(taskIDs, function (tid) { reportTasks(tid,amd$linkModels,codeRule) }) } PretestDescription(appID,uidPrefix,missingHandlers, demographics,statistics,tasks) } ## Write out missing code handers. "processOmitCodes" <- function(handlers,omitCodes) { result <- list() for (handler in handlers) { type <- handler$type code <- omitCodes[type] if (!is.na(code)) { result <- c(result, list(MissingCode(type,"Recode",handler$xmlCode, handler$checksum,code))) } } } reportVariablesAsDemographics <- function(sm, codeRule = NULL) { mod <- getECDIDModel(sm$id) lapply(sm$variables, function (var) { DemographicVariableMeta(paste(mod,var$name,sep="."), "Discrete", translationTable=integerCodes(var$levels, codeRule)) }) } ## Write out the tags corresponding to the tasks "reportTasks" <- function(tid, lmlist,codeRule) { obslist <- list() for (lm in lmlist) { if (lm$taskID == tid) { ##LM for current task for (var in lm$variables) { if (var$role == "Observable") { obslist <- c(obslist, list(ObservationMeta(var$name, translationTable=integerCodes(var$levels, codeRule)))) } } } } names(obslist) <- sapply(obslist, function(o) o$name) TaskMeta(tid,obslist) } "reportTaskEM" <- function(leafid,em,codeRule) { tid <- paste(em$taskID,leafid,".1",sep="") obslist <- list() for (var in em$variables) { if (var$role == "Observable") { obslist <- c(obslist, list(ObservationMeta(var$name, translationTable=integerCodes(var$levels, codeRule)))) } } names(obslist) <- sapply(obslist, function(o) o$name) TaskMeta(tid,obslist) } buildTasksforQmat <- function(Q,amd, codeRule=NULL) { Tasks <- list() for (i in 1:nrow(Q)) { Tasks <- c(Tasks, list(reportTaskEM(Q[i,"TaskName"], amd$evidence[[Q[i,"EvidenceModel"]]], codeRule))) } names(Tasks) <- Q[,"TaskName"] Tasks }