Data generation and analysis modules for standardized questionnaire Usability Metric for User Experience (2010, Kraig Finstad), by amat-design.com.

Sources for items and calculations: Qualaroo

UMUX

Items

Likert scales from 1 to 5.

  1. This software capabilities meet my requirements.
  2. Using this software is a frustrating experience.
  3. This software prototype is easy to use.
  4. I have to spend too much time correcting things with this software.

Generate

UMUX <- list()

n <- 200

for (i in 1:4){
  len <- length(UMUX)
  UMUX[[len+1]] <- numeric(0)
  names(UMUX)[len+1] <- paste("UMUX", i, sep = "")
}

for (i in 1:4) {
  if (i==1 | i==3) {
    UMUX[[i]] <- sample(1:5, n, replace = T, prob = c(1, 1, 1, 3, 2))
  } else if (i==2 | i==4) {
    UMUX[[i]] <- sample(1:5, n, replace = T, prob = c(2, 3, 1, 1, 1))
  }
}

UMUX <- as.data.frame(UMUX)

head(UMUX)
##   UMUX1 UMUX2 UMUX3 UMUX4
## 1     4     4     4     2
## 2     5     2     2     1
## 3     4     4     4     4
## 4     4     1     5     2
## 5     4     1     2     2
## 6     4     2     2     2

Transform

  • Odd items are scored as [user score - 1]. Even items are scored as [5 - user score].
  • Add up these differences and divide the sum by 16 (the highest possible score).
  • Multiply your quotient by 100.
  • Average your results across users.
UMUX.tr <- UMUX

for (i in 1:ncol(UMUX.tr)){
  if (i %% 2 == 0) { #even number
    UMUX.tr[ ,i] <- 5 - UMUX.tr[ ,i]
  } else { #odd
    UMUX.tr[ ,i] <- UMUX.tr[ ,i] - 1
  }
}

UMUX.tr$Score <- rep(0, nrow(UMUX.tr))

for (i in 1:nrow(UMUX.tr)) {
  UMUX.tr[i, "Score"] <- (sum(as.numeric(UMUX.tr[i, ])) / 16) * 100
}

head(UMUX.tr)
##   UMUX1 UMUX2 UMUX3 UMUX4 Score
## 1     3     1     3     3 62.50
## 2     4     3     1     4 75.00
## 3     3     1     3     1 50.00
## 4     3     4     4     3 87.50
## 5     3     4     1     3 68.75
## 6     3     3     1     3 62.50

Infere

# initializing bootstrap
table.S <- numeric(1000)

# loop to generate means from original data
for(i in 1:1000) {
  table.S[i] <- mean(sample(UMUX.tr$Score, 10, replace=T))
}

# sort generated means
table.S.sorted <- sort(table.S)

# catch conf int by selecting heads and tails
UMUX.ci <- c(table.S.sorted[25], table.S.sorted[975])

print(paste("Total mean of Score:", round(mean(UMUX.tr$Score), 1)))
## [1] "Total mean of Score: 64.6"
print(paste("95% CI for SUS Score mean (bootstrap):", round(UMUX.ci, 1)[1], round(UMUX.ci, 1)[2]))
## [1] "95% CI for SUS Score mean (bootstrap): 53.8 74.4"

Visualize

if (!require(ggplot2)) install.packages("ggplot2")
## Loading required package: ggplot2
library(ggplot2)
vlines <- data.frame("Mean" = mean(UMUX.tr$Score))

CIrect <- data.frame("Low" = UMUX.ci[1], "High" = UMUX.ci[2])

limits <- c(0, 100)

ggplot() +
  geom_histogram(data = UMUX.tr, aes(Score, fill = ..x..), binwidth = 8) +
  coord_cartesian(xlim = limits) +
  scale_fill_gradient(name = "Score", low = "red", high = "green", limits = limits) +
  geom_vline(data = vlines, aes(xintercept = Mean), colour = "DodgerBlue") +
  geom_text(data = vlines, aes(x = Mean + 3, label="Mean/CI", y = 4.5), colour = "DodgerBlue", angle = 90) +
  geom_rect(aes(xmin = CIrect$Low, xmax = CIrect$High), ymin = -200, ymax = 200, fill = "DodgerBlue", alpha = 0.15) +
  labs(x = "UMUX scores", y = "Density", title ="Distribution of UMUX scores")

Conclusion

print(paste("Total UMUX score:", round(mean(UMUX.tr$Score), 1)))
## [1] "Total UMUX score: 64.6"

UMUX-LITE

Items

Likert scales from 1 to 5 (or 1 to 7: this last one allows to converting to SUS score).

  1. This software capabilities meet my requirements.
  2. This software prototype is easy to use.

Generate

UMUXlite <- list()

for (i in 1:2){
  len <- length(UMUXlite)
  UMUXlite[[len+1]] <- numeric(0)
  names(UMUXlite)[len+1] <- paste("UMUXlite", i, sep = "")
}

UMUXlite[[1]] <- sample(1:7, n, replace = T, prob = c(0.1, 0.2, 0.5, 1, 3, 8, 4))
UMUXlite[[2]] <- sample(1:7, n, replace = T, prob = c(0.1, 0.2, 0.5, 1, 3, 8, 4))

UMUXlite <- as.data.frame(UMUXlite)

head(UMUXlite)
##   UMUXlite1 UMUXlite2
## 1         7         5
## 2         6         6
## 3         6         6
## 4         6         5
## 5         5         7
## 6         5         6

Transform

  • Items are scored by subtracting one from the user response: [user score - 1]
  • Add the two adjusted scores and divide the sum by 12 (the highest possible score).
  • Multiply your quotient by 100.
  • Average your results across users.
UMUXlite.tr <- UMUXlite

UMUXlite.tr <- UMUXlite.tr - 1

UMUXlite.tr$Score <- rep(0, nrow(UMUXlite.tr))

for (i in 1:nrow(UMUXlite.tr)) {
  UMUXlite.tr[i, "Score"] <- (sum(as.numeric(UMUXlite.tr[i, ])) / 12) * 100
}

head(UMUXlite.tr)
##   UMUXlite1 UMUXlite2    Score
## 1         6         4 83.33333
## 2         5         5 83.33333
## 3         5         5 83.33333
## 4         5         4 75.00000
## 5         4         6 83.33333
## 6         4         5 75.00000

Infere

# initializing bootstrap
table.S <- numeric(1000)

# loop to generate means from original data
for(i in 1:1000) {
  table.S[i] <- mean(sample(UMUXlite.tr$Score, 10, replace=T))
}

# sort generated means
table.S.sorted <- sort(table.S)

# catch conf int by selecting heads and tails
UMUXlite.ci <- c(table.S.sorted[25], table.S.sorted[975])

print(paste("Total mean of Score:", round(mean(UMUXlite.tr$Score), 1)))
## [1] "Total mean of Score: 80.4"
print(paste("95% CI for SUS Score mean (bootstrap):", round(UMUXlite.ci, 1)[1], round(UMUXlite.ci, 1)[2]))
## [1] "95% CI for SUS Score mean (bootstrap): 72.5 87.5"

SUS score prediction

UMUXlite.tr$SUScomp <- rep(0, nrow(UMUXlite.tr))

for (i in 1:nrow(UMUXlite.tr)) {
  UMUXlite.tr[i, "SUScomp"] <- 
    0.65 * ((UMUXlite.tr[i, 1] + UMUXlite.tr[i, 2] - 2) * (100/12)) + 22.9
}

print(paste("SUS comparison score:", round(mean(UMUXlite.tr$SUScomp), 1)))
## [1] "SUS comparison score: 64.3"

Visualize

vlines <- data.frame("Mean" = mean(UMUXlite.tr$Score))

CIrect <- data.frame("Low" = UMUXlite.ci[1], "High" = UMUXlite.ci[2])

limits <- c(0, 100)

ggplot() +
  geom_histogram(data = UMUXlite.tr, aes(Score, fill = ..x..), binwidth = 8) +
  coord_cartesian(xlim = limits) +
  scale_fill_gradient(name = "Score", low = "red", high = "green", limits = limits) +
  geom_vline(data = vlines, aes(xintercept = Mean), colour = "DodgerBlue") +
  geom_text(data = vlines, aes(x = Mean + 3, label="Mean/CI", y = 4.5), colour = "DodgerBlue", angle = 90) +
  geom_rect(aes(xmin = CIrect$Low, xmax = CIrect$High), ymin = -200, ymax = 200, fill = "DodgerBlue", alpha = 0.15) +
  labs(x = "UMUX Lite scores", y = "Density", title ="Distribution of UMUX Lite scores")

Conclusion

print(paste("Total UMUX-Lite score:", round(mean(UMUXlite.tr$Score), 1)))
## [1] "Total UMUX-Lite score: 80.4"
print(paste("SUS comparison score:", round(mean(UMUXlite.tr$SUScomp), 1)))
## [1] "SUS comparison score: 64.3"