Data generation and analysis modules for standardized questionnaire System Usability Scale (J. Brooke, 1986).

Made by amat-design.com.

Sources for items and calculations: Stuart Cunningham, UXPA Journal, Bangor, Korter, Miller

Items

Likert scales from 1 to 5, from Strongly disagree to Strongly agree:

  1. I think that I would like to use this system frequently.
  2. I found the system unnecessarily complex.
  3. I thought the system was easy to use.
  4. I think that I would need the support of a technical person to be able to use this system.
  5. I found the various functions in this system were well integrated.
  6. I thought there was too much inconsistency in this system.
  7. I would imagine that most people would learn to use this system very quickly.
  8. I found the system very cumbersome to use.
  9. I felt very confident using the system.
  10. I needed to learn a lot of things before I could get going with this system.

Themes:

  1. Learnability (items 4 and 10)
  2. Usability (other items: items 1, 2, 3, 5, 6, 7, 8, 9)

Generate

# initiate SUS variable
SUS <- list()

# initiate items variables
for (i in 1:10){
  len <- length(SUS)
  SUS[[len+1]] <- NA
  names(SUS)[len+1] <- paste("SUS", i, sep = "")
}

# set number of participants
n <- 200

prob <- c(1, 1, 1, 10, 10)

# populate SUS
for (i in 1:10) {
  if (i==1 | i==3 | i==5 | i==7 | i==9) {
    SUS[[i]] <- sample(1:5, n, replace = T, prob = prob)
  } else if (i==2 | i==4 | i==6 | i==8 | i==10) {
    SUS[[i]] <- sample(5:1, n, replace = T, prob = prob)
  }
}

# make it a data frame
SUS <- as.data.frame(SUS)

# display 6 first rows
head(SUS)
##   SUS1 SUS2 SUS3 SUS4 SUS5 SUS6 SUS7 SUS8 SUS9 SUS10
## 1    5    2    3    1    5    2    4    3    5     1
## 2    4    2    2    1    4    1    4    2    4     1
## 3    4    1    5    1    5    1    2    2    5     1
## 4    5    1    4    1    5    1    4    2    4     1
## 5    4    1    4    1    5    2    5    1    3     1
## 6    4    2    4    1    3    1    4    1    5     2

Transform

Compute total SUS score:

  • For odd items: subtract one from the user response.
  • For even-numbered items: subtract the user responses from 5. This scales all values from 0 to 4 (with four being the most positive response).
  • Add up the converted responses for each user and multiply that total by 2.5. This converts the range of possible values from 0 to 100 instead of from 0 to 40.

Compute score by themes:

Note: didn’t find official formula to compute learnability and usability. I applied the same protocol but multiplying buy 12.5 for learnability and 3.125 for usability to convert at same range as overall scoring.

Interpretation of SUS scores (grades):

  • 0-60: F
  • 61-70: D
  • 71-80: C
  • 81-90: B
  • 91-100: A

Interpretation of SUS scores (acceptability):

  • 0-50: Not acceptable
  • 51-62: Marginal low
  • 63-70: Marginal high
  • 71-100: Acceptable

The market average SUS score is 68.

market.SUS <- 68

SUS.tr <- SUS

# Scale values from 0 to 4
for (i in 1:length(SUS)) {
  if (i %% 2 == 0) { #even number
    SUS.tr[[i]] <- 5 - SUS[[i]]
  } else { #odd
    SUS.tr[[i]] <- SUS[[i]] - 1
  }
}

# initiate a Score by row variable
SUS.tr$Score <- rep(0, nrow(SUS.tr))

# sum responses by row and multiply it by 2.5
for(i in 1:nrow(SUS.tr)) {
  SUS.tr[i, "Score"] <- sum(as.numeric(SUS.tr[i, 1:10])) * 2.5
}

# initiate a Grade by row variable
SUS.tr$Grade <- rep(0, nrow(SUS.tr))

# function to compute grade from score
sus.grade <- function(x) {
  if (x >= 91) {
    return("A")
  } else if (x >= 81) {
      return("B")
  } else if (x >= 71) {
      return("C")
  } else if (x >= 61) {
      return("D")
  } else {
      return("F")
  }
}

# apply function to each row
for(i in 1:nrow(SUS.tr)) {
  SUS.tr[i, "Grade"] <- sus.grade(SUS.tr[i, "Score"])
}

# make this variable a factor and order levels
SUS.tr$Grade <- factor(SUS.tr$Grade, levels = c("F", "D", "C", "B", "A"))

# initiate an Acceptability variable
SUS.tr$Acceptability <- rep(0, nrow(SUS.tr))

# function to compute variability from score
sus.acceptability <- function(x) {
  if (x >= 71) {
    return("Acceptable")
  } else if (x >= 63) {
      return("Marginal high")
  } else if (x >= 50) {
      return("Marginal low")
  } else {
      return("Not acceptable")
  }
}

# apply function for each row
for(i in 1:nrow(SUS.tr)) {
  SUS.tr[i, "Acceptability"] <- sus.acceptability(SUS.tr[i, "Score"])
}

# make acceptability a factor and order levels
SUS.tr$Acceptability <- factor(SUS.tr$Acceptability, levels = c("Not acceptable", "Marginal low", "Marginal high", "Acceptable"))

# initiate a Learnability variable
SUS.tr$Learnability <- rep(0, nrow(SUS.tr))

# compute learnability by row
for(i in 1:nrow(SUS.tr)) {
  SUS.tr[i, "Learnability"] <- sum(as.numeric(SUS.tr[i, c(4, 10)])) * (100/(4*2))
}

# initiate a Usability variable
SUS.tr$Usability <- rep(0, nrow(SUS.tr))

# compute Usability by row
for(i in 1:nrow(SUS.tr)) {
  SUS.tr[i, "Usability"] <- sum(as.numeric(SUS.tr[i, c(1:3, 5:9)])) * (100/(4*8))
}

# display 6 first rows
head(SUS.tr)
##   SUS1 SUS2 SUS3 SUS4 SUS5 SUS6 SUS7 SUS8 SUS9 SUS10 Score Grade Acceptability
## 1    4    3    2    4    4    3    3    2    4     4  82.5     B    Acceptable
## 2    3    3    1    4    3    4    3    3    3     4  77.5     C    Acceptable
## 3    3    4    4    4    4    4    1    3    4     4  87.5     B    Acceptable
## 4    4    4    3    4    4    4    3    3    3     4  90.0     B    Acceptable
## 5    3    4    3    4    4    3    4    4    2     4  87.5     B    Acceptable
## 6    3    3    3    4    2    4    3    4    4     3  82.5     B    Acceptable
##   Learnability Usability
## 1        100.0    78.125
## 2        100.0    71.875
## 3        100.0    84.375
## 4        100.0    87.500
## 5        100.0    84.375
## 6         87.5    81.250

Infere

# initializing bootstrap
table.S <- numeric(1000)

# loop to generate means from original data
for(i in 1:1000) {
  table.S[i] <- mean(sample(SUS.tr$Score, 10, replace=T))
}

# sort generated means
table.S.sorted <- sort(table.S)

# catch conf int by selecting heads and tails
SUS.ci <- c(table.S.sorted[25], table.S.sorted[975])

print(paste("Total mean of Score:", round(mean(SUS.tr$Score), 1)))
## [1] "Total mean of Score: 80.1"
print(paste("95% CI for SUS Score mean (bootstrap):", round(SUS.ci, 1)[1], round(SUS.ci, 1)[2]))
## [1] "95% CI for SUS Score mean (bootstrap): 74.8 84.5"

Visualize

if (!require(ggplot2)) install.packages("ggplot2")
## Loading required package: ggplot2
library(ggplot2)
# Scores distribution
vlines <- data.frame("Mean" = mean(SUS.tr$Score), "Market" = market.SUS)

CIrect <- data.frame("Low" = SUS.ci[1], "High" = SUS.ci[2])

limits <- c(0, 100)

ggplot() +
  geom_histogram(data = SUS.tr, aes(Score, fill = ..x..), binwidth = 2.5) + 
  coord_cartesian(xlim = limits) +
  scale_fill_gradient(name = "Score", low = "red", high = "green", limits = limits) +
  geom_vline(data = vlines, aes(xintercept = Mean), colour = "DodgerBlue") +
  geom_text(data = vlines, aes(x = Mean + 2, label="Mean/CI", y = 2.5), colour = "DodgerBlue", angle = 90) +
  geom_vline(data = vlines, aes(xintercept = Market), linetype = "dashed", colour = "LightCoral") +
  geom_text(data = vlines, aes(x = Market + 2, label = "Market", y = 2), colour = "LightCoral", angle = 90) +
  geom_rect(aes(xmin = CIrect$Low, xmax = CIrect$High), ymin = -200, ymax = 200, fill = "DodgerBlue", alpha = 0.15) +
  labs(x = "SUS scores", y = "Density", title ="Distribution of SUS scores", subtitle = "With market average comparison")

# Interpretation of SUS scores (grades):
# 
# -   0-60: F
# -   61-70: D
# -   71-80: C
# -   81-90: B
# -   91-100: A
# 
# Interpretation of SUS scores (acceptability):
# 
# -   0-50: Not acceptable
# -   51-62: Marginal low
# -   63-70: Marginal high
# -   71-100: Acceptable
# 
# The average SUS score is 68.

xmin <- c(0, 61, 71, 81, 91)
xmax <- c(61, 71, 81, 91, 100)
label <- c("F", "D", "C", "B", "A")
ymin <- rep(0.5, 5)
ymax <- rep(1.5, 5)

grades <- data.frame(xmin, xmax, label, ymin, ymax)

xmin <- c(0, 51, 63, 71)
xmax <- c(51, 63, 71, 100)
label <- c("Not acceptable", "Marginal low", "Marginal high", "Acceptable")
ymin <- rep(2, 4)
ymax <- rep(3, 4)

accept <- data.frame(xmin, xmax, label, ymin, ymax)

ggplot() +
  coord_cartesian(xlim = c(0, 100), ylim = c(0.25, 3.25)) +
  geom_rect(data = grades, mapping = aes(xmin = xmin, xmax = xmax, ymin = ymin, ymax = ymax, fill = label), alpha = 0.40) +
  geom_text(data = grades, mapping = aes(x = xmax - 4, y = ymin + 0.25, label = label), alpha = 0.5) +
  geom_rect(data = accept, mapping = aes(xmin = xmin, xmax = xmax, ymin = ymin, ymax = ymax, fill = label), alpha = 0.40) +
  geom_text(data = accept, mapping = aes(x = xmax - 4, y = ymin + 0.05, label = label), angle = 90, hjust = 0, nudge_y = 0.1, alpha = 0.5) +
  scale_fill_brewer(palette = c("RdBu"), direction = -1) +
  scale_color_brewer(palette = c("RdBu"), direction = -1) +
  geom_vline(data = vlines, aes(xintercept = Mean), colour = "DodgerBlue", size = 0.75) +
  geom_text(data = vlines, aes(x = Mean + 2, label="Mean/CI", y = 0), colour = "DodgerBlue", angle = 90, nudge_y = 0.25, hjust = 0) +
  geom_vline(data = vlines, aes(xintercept = Market), linetype = "dashed", colour = "LightCoral", size = 0.75) +
  geom_text(data = vlines, aes(x = Market + 2, label = "Market", y = 0), colour = "LightCoral", angle = 90, nudge_y = 0.25, hjust = 0) +
  geom_rect(aes(xmin = CIrect$Low, xmax = CIrect$High), ymin = -200, ymax = 200, fill = "DodgerBlue", alpha = 0.15) +
  theme_minimal() +
  theme(axis.text.y = element_blank(), axis.ticks.y = element_blank(), legend.position = "none", panel.grid.major = element_blank(), panel.grid.minor = element_blank()) +
  labs(title ="SUS mean score interpretation", subtitle = "with confidence interval", x = "Mean score", y = " ")

Conclusion

print(paste("Total mean score:", round(mean(SUS.tr$Score), 1)))
## [1] "Total mean score: 80.1"
if (mean(SUS.tr$Score) > market.SUS) {
  print(paste("Total mean score is above average SUS score for other products (", market.SUS, ").", sep = ""))
} else {
  print("Total mean score is below average SUS score for other products.")
}
## [1] "Total mean score is above average SUS score for other products (68)."
print(paste("Grade of total mean score:", sus.grade(mean(SUS.tr$Score))))
## [1] "Grade of total mean score: C"
print(paste("Acceptability of total mean score:", sus.acceptability(mean(SUS.tr$Score))))
## [1] "Acceptability of total mean score: Acceptable"
print(paste("Learnability mean score:", round(mean(SUS.tr$Learnability), 1)))
## [1] "Learnability mean score: 80.6"
print(paste("Usability mean score:", round(mean(SUS.tr$Usability), 1)))
## [1] "Usability mean score: 80"