## Warning: package 'matrixStats' was built under R version 4.4.1
## Warning: package 'S4Vectors' was built under R version 4.4.1
## Warning: package 'IRanges' was built under R version 4.4.1

1 Introduction

Paired-end sequencing was performed on primary cultures from parathyroid tumors of 4 patients at 2 time points over 3 conditions (control, treatment with diarylpropionitrile (DPN) and treatment with 4-hydroxytamoxifen (OHT)). DPN is a selective estrogen receptor agonist and OHT is a selective estrogen receptor modulator. One sample (patient 4, 24 hours, control) was omitted by the paper authors due to low quality. Data, the count table and information on the experiment is available at http://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE37211.

#Count data and meta data

data("parathyroidGenesSE", package="parathyroidSE")
se1 <- parathyroidGenesSE
rm(parathyroidGenesSE)
dupExps <- colData(se1) %>% 
  as.data.frame() %>% 
  filter(duplicated(experiment))  %>% 
  pull(experiment)

counts <- assays(se1)$counts
newCounts <- counts
cd <- colData(se1)
for(ss in 1:length(dupExps)){
  # check which samples are duplicates
  relevantId <- which(colData(se1)$experiment == dupExps[ss])
  # sum counts
  newCounts[,relevantId[1]] <- rowSums(counts[,relevantId])
  # keep which columns / rows to remove.
  if(ss == 1){
    toRemove <- relevantId[2]
  } else {
    toRemove <- c(toRemove, relevantId[2])
  }
}

# remove after summing counts (otherwise IDs get mixed up)
newCounts <- newCounts[,-toRemove]
newCD <- cd[-toRemove,]

# Create new SummarizedExperiment
se <- SummarizedExperiment(assays = list("counts" = newCounts),
                            colData = newCD,
                            metadata = metadata(se1))
rm(se1)

2 Data analysis

2.1 Count object

dge <- DGEList(counts=assay(se))
dge$sample

2.2 Design

There can be an effect of agent, time interaction and agent x time interaction. We also expect blocking for patient. We can assess all effects of interest within patient.

design <- model.matrix(~time*treatment+patient,colData(se))
rownames(design) = colnames(dge)
design
##          (Intercept) time48h treatmentDPN treatmentOHT patient2 patient3
## Sample1            1       0            0            0        0        0
## Sample2            1       1            0            0        0        0
## Sample3            1       0            1            0        0        0
## Sample4            1       1            1            0        0        0
## Sample5            1       0            0            1        0        0
## Sample6            1       1            0            1        0        0
## Sample7            1       0            0            0        1        0
## Sample8            1       1            0            0        1        0
## Sample9            1       0            1            0        1        0
## Sample10           1       1            1            0        1        0
## Sample11           1       0            0            1        1        0
## Sample12           1       1            0            1        1        0
## Sample13           1       0            0            0        0        1
## Sample14           1       1            0            0        0        1
## Sample15           1       0            1            0        0        1
## Sample16           1       1            1            0        0        1
## Sample17           1       0            0            1        0        1
## Sample18           1       1            0            1        0        1
## Sample19           1       1            0            0        0        0
## Sample20           1       0            1            0        0        0
## Sample21           1       1            1            0        0        0
## Sample22           1       0            0            1        0        0
## Sample23           1       1            0            1        0        0
##          patient4 time48h:treatmentDPN time48h:treatmentOHT
## Sample1         0                    0                    0
## Sample2         0                    0                    0
## Sample3         0                    0                    0
## Sample4         0                    1                    0
## Sample5         0                    0                    0
## Sample6         0                    0                    1
## Sample7         0                    0                    0
## Sample8         0                    0                    0
## Sample9         0                    0                    0
## Sample10        0                    1                    0
## Sample11        0                    0                    0
## Sample12        0                    0                    1
## Sample13        0                    0                    0
## Sample14        0                    0                    0
## Sample15        0                    0                    0
## Sample16        0                    1                    0
## Sample17        0                    0                    0
## Sample18        0                    0                    1
## Sample19        1                    0                    0
## Sample20        1                    0                    0
## Sample21        1                    1                    0
## Sample22        1                    0                    0
## Sample23        1                    0                    1
## attr(,"assign")
## [1] 0 1 2 2 3 3 3 4 4
## attr(,"contrasts")
## attr(,"contrasts")$time
## [1] "contr.treatment"
## 
## attr(,"contrasts")$treatment
## [1] "contr.treatment"
## 
## attr(,"contrasts")$patient
## [1] "contr.treatment"
ExploreModelMatrix::VisualizeDesign(colData(se),~ time*treatment + patient)$plotlist
## $`time = 24h`

## 
## $`time = 48h`

2.3 Filtering

keep <- filterByExpr(dge,design)
table(keep)
## keep
## FALSE  TRUE 
## 46629 16564
dge <- dge[keep, , keep.lib.sizes=FALSE]

2.4 Normalisation

dge <- calcNormFactors(dge)
dge$samples

2.5 Data exploration

An MDS plot shows the leading fold changes (differential expression) between the 23 samples.

plotMDS(dge,labels=paste(colData(se)$treatment,colData(se)$time,colData(se)$patient,sep="-"),col=as.double(colData(se)$treatment))

There is a very strong patient effect! To further assess the treatment effects we can make MDS plots per patient

for (i in 1:4)
plotMDS(dge[,colData(se)$patient==i], col=as.double(colData(se)$treatment)[colData(se)$patient==i],
labels=paste(colData(se)$treatment[colData(se)$patient==i],
             colData(se)$time[colData(se)$patient==i],
             colData(se)$patient,sep="-")[colData(se)$patient==i])

2.6 Parameter estimation

We will use the default Quasi likelihood approach of edgeR.

For quasi-likelihood we do not specify the full distribution, only the first two moments: the mean and the variance, which is sufficient to do inference on the mean.

\[ \left\{ \begin{array}{lcl} E[y_{ig}\vert \mathbf{x}_{ig}]&=&\mu_{ig}\\ log(\mu_{ig})&=&\eta_{ig}\\ \eta_{ig}&=&\beta_0 + \beta_{t2} x_{t2,i} + \beta_{DPN} x_{DPN,i} + \beta_\text{DPN:t2} x_{DPN,i}x_{t2,i} \\ && \quad + \beta_{OHT}x_{OHT,i} + \beta_\text{OHT:t2} x_{OHT,i}x_{t2,i} \\ && \quad + \beta_{p2}x_{p2,i} + \beta_{p3}x_{p3,i} + \beta_{p4}x_{p4,i}\ + \log N_i\\ \text{Var}[y_{ig}\vert \mathbf{x}_{ig}]&=&\sigma^2_g\left(\mu_{ig}+\phi\mu_{ig}^2\right) \end{array}\right. \]

with \(\sigma^2_g\) an additional dispersion parameter the scales the negative binomial variance function, \(x_{DPN,i}\), \(x_{DPN,i}\), \(x_{t2,i}\), \(x_{p.,i}\) dummy variables that is 1 if cell line was treated with DPN, OHT, incubated for 48 h, from patient \(p.\), respectively and is 0 otherwise, and, \(\log{N}_i\) a normalisation offset to correct for sequencing depth. Note, that \(\beta_{DPN}\) is the main effect for the DPN treatment, and corresponds to the average log fold change between treated and control mice after 24h. The interaction \(\beta_\text{DPN:t2}\) can be interpreted as the average change in log2 FC between DPN treated and control cell lines at the late and early timepoint. The researchers are also interested in a assessing third contrast: the effect of the DPN treatment at the late time point.

\[ \log_2\text{FC}^\text{48h}_\text{DPN - C}= \beta_{DPN}+\beta_{DPN,t2}\]

For the OHT treatment we will assess similar contrasts.

\[ \log_2\text{FC}^\text{24h}_\text{OHT - C}= \beta_{OHT}\] \[ \log_2\text{FC}^\text{48h}_\text{OHT - C}= \beta_{OHT}+\beta_{OHT,t2}\] \[ \log_2\text{FC}^\text{48h}_\text{OHT - C} -\log_2\text{FC}^\text{24h}_\text{OHT - C}= \beta_{OHT,t2} \]

Finally, we also have to assess if there is a difference between DPN and OHT treatment

\[ \log_2\text{FC}^\text{24h}_\text{OHT - DPN}= \beta_{OHT} - \beta_{DPN}\] \[ \log_2\text{FC}^\text{48h}_\text{OHT - C}= \beta_{OHT}+\beta_{OHT,t2} - \beta_{DPN} - \beta_{DPN,t2} \]

\[ \log_2\text{FC}^\text{48h}_\text{OHT - DPN} -\log_2\text{FC}^\text{24h}_\text{OHT - DPN}= \beta_{OHT,t2} - \beta_{DPN,t2} \]

dge <- estimateDisp(dge, design)  
plotBCV(dge)

The quasi-negative binomial model can be fitted using the function glmQLFit

fit <- glmQLFit(dge,design)

2.7 Contrasts

We now implement all 9 contrasts of interest

L <- msqrob2::makeContrast(
    c("treatmentDPN = 0",
      "treatmentOHT = 0",
      "treatmentOHT - treatmentDPN = 0", 
      "treatmentDPN + time48h:treatmentDPN = 0",
      "treatmentOHT + time48h:treatmentOHT = 0",
      "treatmentOHT + time48h:treatmentOHT - treatmentDPN - time48h:treatmentDPN = 0", 
      "time48h:treatmentDPN = 0", 
      "time48h:treatmentOHT = 0", 
      "time48h:treatmentOHT - time48h:treatmentDPN = 0"),
    parameterNames = colnames(design))
L
##                      treatmentDPN treatmentOHT treatmentOHT - treatmentDPN
## (Intercept)                     0            0                           0
## time48h                         0            0                           0
## treatmentDPN                    1            0                          -1
## treatmentOHT                    0            1                           1
## patient2                        0            0                           0
## patient3                        0            0                           0
## patient4                        0            0                           0
## time48h:treatmentDPN            0            0                           0
## time48h:treatmentOHT            0            0                           0
##                      treatmentDPN + time48h:treatmentDPN
## (Intercept)                                            0
## time48h                                                0
## treatmentDPN                                           1
## treatmentOHT                                           0
## patient2                                               0
## patient3                                               0
## patient4                                               0
## time48h:treatmentDPN                                   1
## time48h:treatmentOHT                                   0
##                      treatmentOHT + time48h:treatmentOHT
## (Intercept)                                            0
## time48h                                                0
## treatmentDPN                                           0
## treatmentOHT                                           1
## patient2                                               0
## patient3                                               0
## patient4                                               0
## time48h:treatmentDPN                                   0
## time48h:treatmentOHT                                   1
##                      treatmentOHT + time48h:treatmentOHT - treatmentDPN - time48h:treatmentDPN
## (Intercept)                                                                                  0
## time48h                                                                                      0
## treatmentDPN                                                                                -1
## treatmentOHT                                                                                 1
## patient2                                                                                     0
## patient3                                                                                     0
## patient4                                                                                     0
## time48h:treatmentDPN                                                                        -1
## time48h:treatmentOHT                                                                         1
##                      time48h:treatmentDPN time48h:treatmentOHT
## (Intercept)                             0                    0
## time48h                                 0                    0
## treatmentDPN                            0                    0
## treatmentOHT                            0                    0
## patient2                                0                    0
## patient3                                0                    0
## patient4                                0                    0
## time48h:treatmentDPN                    1                    0
## time48h:treatmentOHT                    0                    1
##                      time48h:treatmentOHT - time48h:treatmentDPN
## (Intercept)                                                    0
## time48h                                                        0
## treatmentDPN                                                   0
## treatmentOHT                                                   0
## patient2                                                       0
## patient3                                                       0
## patient4                                                       0
## time48h:treatmentDPN                                          -1
## time48h:treatmentOHT                                           1

2.8 Tests

We have to perform a quasi- F-test for each contrast. The quasi F-test involves fitting a different model for each contrast so that we can compare the full model with a reduced model that implies that one specific contrast is zero.

Because we estimated the additional dispersion parameter \(\sigma^2_g\) using a sum of squared deviance residuals:

i.e.  \[e_{i,d} = 2 (l_i(y_i,y_i) - l_i(\mu_i,y_i)\] and

\[ \hat \sigma_g^2 = \frac{\sum_{i=1}^n e_{i,d}^2}{n-p} = \frac{2\left[l(\mathbf{y},\mathbf{y}) - l(\boldsymbol{\mu},\mathbf{y})\right]}{n-p} \]

With edgeR we will then further adopt empirical Bayes to borrow strength across genes to stabilise the parameter estimator, which will also increase the degrees if this gene wise dispersion parameter estimator which we refer to as \(df_\text{res}^{EB}\).

We can use a quasi F -test that can also correct for the degrees of freedom that have been used to estimate mean model parameters and the residual degrees of freedom that were available for estimating the additional dispersion parameter. The quasi F test will thus perform better in a small sample setting. It is defined as:

\[ F = \frac{\frac{LRT_\text{g, full - reduced}}{df_{LRT}}}{\sigma^2_g} \]

which follows an F - distribution with \(df_{LRT}\) degrees of freedom in the nominator and \(df_{res}^{EB}\) degrees of freedom in the denominator under the null hypothesis that the full and reduced model are equivalent and that the assessed contrasts are thus equal to zero. Indeed, the dispersion estimator in the denominator follows a scaled \(\chi^2\) distribution with \(df_{res}^{EB}\) degrees of freedom.

We perform all tests and loop over the columns of L for this purpose.

testsF <- apply(L, 2, function(fit,contrast) 
    glmQLFTest(fit,contrast=contrast), 
    fit = fit)
topTablesF<- lapply(testsF, topTags, n=nrow(dge))

sapply(topTablesF, function(x) sum(x$table$FDR <0.05))
##                                                              treatmentDPN 
##                                                                         0 
##                                                              treatmentOHT 
##                                                                         0 
##                                               treatmentOHT - treatmentDPN 
##                                                                         0 
##                                       treatmentDPN + time48h:treatmentDPN 
##                                                                         5 
##                                       treatmentOHT + time48h:treatmentOHT 
##                                                                        11 
## treatmentOHT + time48h:treatmentOHT - treatmentDPN - time48h:treatmentDPN 
##                                                                         0 
##                                                      time48h:treatmentDPN 
##                                                                         0 
##                                                      time48h:treatmentOHT 
##                                                                         0 
##                               time48h:treatmentOHT - time48h:treatmentDPN 
##                                                                         0

We only find significant fold changes for very few genes between OHT and the control treatment at the late time point. We also did not find significant interactions.

3 Plots

3.1 Volcano plots

for (i in 1:ncol(L))
{
   volcano<- ggplot(topTablesF[[i]]$table,aes(x=logFC,y=-log10(PValue),color=FDR < 0.05)) + geom_point() + scale_color_manual(values=c("black","red")) + ggtitle(paste("contrast",names(topTablesF)[i]))
print(volcano)
}

3.2 Histograms of p-values

histsP <- lapply(topTablesF, function(x) 
    x$table %>% 
        ggplot(aes(x=PValue)) + 
        geom_histogram(breaks =seq(0,1,.1) ,col=1)
    )
    
for (i in 1:ncol(L)) 
    histsP[[i]] <- histsP[[i]] +
    ggtitle(paste("contrast",names(topTablesF)[i]))
histsP
## $treatmentDPN

## 
## $treatmentOHT

## 
## $`treatmentOHT - treatmentDPN`

## 
## $`treatmentDPN + time48h:treatmentDPN`

## 
## $`treatmentOHT + time48h:treatmentOHT`

## 
## $`treatmentOHT + time48h:treatmentOHT - treatmentDPN - time48h:treatmentDPN`

## 
## $`time48h:treatmentDPN`

## 
## $`time48h:treatmentOHT`

## 
## $`time48h:treatmentOHT - time48h:treatmentDPN`

3.3 heatmaps

for (i in 1:ncol(L))
{
    sigID <- topTablesF[[i]]$table %>% 
        filter(FDR<0.05) %>% 
        rownames
    if (length(sigID)>0)
        heatmap(dge$counts[sigID,], main = colnames(L)[i], cex.main=.2)
}

4 EdgeR traditional

fitGlm <- glmFit(dge,design)
testLRT2 <- apply(L, 2, function(fit,contrast) 
    glmLRT(fit,contrast=contrast), 
    fit = fitGlm)
topTablesLRT2 <- lapply(testLRT2, topTags, n=nrow(dge))
sapply(topTablesLRT2, function(x) sum(x$table$FDR <0.05))
##                                                              treatmentDPN 
##                                                                         2 
##                                                              treatmentOHT 
##                                                                         0 
##                                               treatmentOHT - treatmentDPN 
##                                                                         4 
##                                       treatmentDPN + time48h:treatmentDPN 
##                                                                        64 
##                                       treatmentOHT + time48h:treatmentOHT 
##                                                                        23 
## treatmentOHT + time48h:treatmentOHT - treatmentDPN - time48h:treatmentDPN 
##                                                                        11 
##                                                      time48h:treatmentDPN 
##                                                                         0 
##                                                      time48h:treatmentOHT 
##                                                                         0 
##                               time48h:treatmentOHT - time48h:treatmentDPN 
##                                                                         0

We find more genes for the traditional edgeR workflow, however, it is known that this workflow is often too liberal.

LS0tCnRpdGxlOiAiUGFyYXRoeXJvaWQ6IERFIGFuYWx5c2lzIgphdXRob3I6ICJMaWV2ZW4gQ2xlbWVudCIKZGF0ZTogInN0YXRPbWljcywgR2hlbnQgVW5pdmVyc2l0eSAoaHR0cHM6Ly9zdGF0b21pY3MuZ2l0aHViLmlvKSIKb3V0cHV0OgogICAgaHRtbF9kb2N1bWVudDoKICAgICAgY29kZV9kb3dubG9hZDogdHJ1ZQogICAgICB0aGVtZTogZmxhdGx5CiAgICAgIHRvYzogdHJ1ZQogICAgICB0b2NfZmxvYXQ6IHRydWUKICAgICAgaGlnaGxpZ2h0OiB0YW5nbwogICAgICBudW1iZXJfc2VjdGlvbnM6IHRydWUKICAgIHBkZl9kb2N1bWVudDoKICAgICAgdG9jOiB0cnVlCiAgICAgIG51bWJlcl9zZWN0aW9uczogdHJ1ZQpsaW5rY29sb3I6IGJsdWUKdXJsY29sb3I6IGJsdWUKY2l0ZWNvbG9yOiBibHVlCi0tLQoKCgpgYGB7cixlY2hvPUZBTFNFfQpzdXBwcmVzc1BhY2thZ2VTdGFydHVwTWVzc2FnZXMoewpsaWJyYXJ5KGVkZ2VSKQpsaWJyYXJ5KFN1bW1hcml6ZWRFeHBlcmltZW50KQpsaWJyYXJ5KHRpZHl2ZXJzZSkKICBpZighInBhcmF0aHlyb2lkU0UiICVpbiUgaW5zdGFsbGVkLnBhY2thZ2VzKClbLDFdKSBCaW9jTWFuYWdlcjo6aW5zdGFsbCgicGFyYXRoeXJvaWRTRSIpCn0pCgpgYGAKCiMgSW50cm9kdWN0aW9uCgpQYWlyZWQtZW5kIHNlcXVlbmNpbmcgd2FzIHBlcmZvcm1lZCBvbiBwcmltYXJ5IGN1bHR1cmVzIGZyb20gcGFyYXRoeXJvaWQgdHVtb3JzIG9mIDQgcGF0aWVudHMgYXQgMiB0aW1lIHBvaW50cyBvdmVyIDMgY29uZGl0aW9ucyAoY29udHJvbCwgdHJlYXRtZW50IHdpdGggZGlhcnlscHJvcGlvbml0cmlsZSAoRFBOKSBhbmQgdHJlYXRtZW50IHdpdGggNC1oeWRyb3h5dGFtb3hpZmVuIChPSFQpKS4gRFBOIGlzIGEgc2VsZWN0aXZlIGVzdHJvZ2VuIHJlY2VwdG9yIGFnb25pc3QgYW5kIE9IVCBpcyBhIHNlbGVjdGl2ZSBlc3Ryb2dlbiByZWNlcHRvciBtb2R1bGF0b3IuIE9uZSBzYW1wbGUgKHBhdGllbnQgNCwgMjQgaG91cnMsIGNvbnRyb2wpIHdhcyBvbWl0dGVkIGJ5IHRoZSBwYXBlciBhdXRob3JzIGR1ZSB0byBsb3cgcXVhbGl0eS4gRGF0YSwgdGhlIGNvdW50IHRhYmxlIGFuZCBpbmZvcm1hdGlvbiBvbiB0aGUgZXhwZXJpbWVudCBpcyBhdmFpbGFibGUgYXQgaHR0cDovL3d3dy5uY2JpLm5sbS5uaWguZ292L2dlby9xdWVyeS9hY2MuY2dpP2FjYz1HU0UzNzIxMS4KCiNDb3VudCBkYXRhIGFuZCBtZXRhIGRhdGEKCmBgYHtyfQpkYXRhKCJwYXJhdGh5cm9pZEdlbmVzU0UiLCBwYWNrYWdlPSJwYXJhdGh5cm9pZFNFIikKc2UxIDwtIHBhcmF0aHlyb2lkR2VuZXNTRQpybShwYXJhdGh5cm9pZEdlbmVzU0UpCmR1cEV4cHMgPC0gY29sRGF0YShzZTEpICU+JSAKICBhcy5kYXRhLmZyYW1lKCkgJT4lIAogIGZpbHRlcihkdXBsaWNhdGVkKGV4cGVyaW1lbnQpKSAgJT4lIAogIHB1bGwoZXhwZXJpbWVudCkKCmNvdW50cyA8LSBhc3NheXMoc2UxKSRjb3VudHMKbmV3Q291bnRzIDwtIGNvdW50cwpjZCA8LSBjb2xEYXRhKHNlMSkKZm9yKHNzIGluIDE6bGVuZ3RoKGR1cEV4cHMpKXsKICAjIGNoZWNrIHdoaWNoIHNhbXBsZXMgYXJlIGR1cGxpY2F0ZXMKICByZWxldmFudElkIDwtIHdoaWNoKGNvbERhdGEoc2UxKSRleHBlcmltZW50ID09IGR1cEV4cHNbc3NdKQogICMgc3VtIGNvdW50cwogIG5ld0NvdW50c1sscmVsZXZhbnRJZFsxXV0gPC0gcm93U3Vtcyhjb3VudHNbLHJlbGV2YW50SWRdKQogICMga2VlcCB3aGljaCBjb2x1bW5zIC8gcm93cyB0byByZW1vdmUuCiAgaWYoc3MgPT0gMSl7CiAgICB0b1JlbW92ZSA8LSByZWxldmFudElkWzJdCiAgfSBlbHNlIHsKICAgIHRvUmVtb3ZlIDwtIGModG9SZW1vdmUsIHJlbGV2YW50SWRbMl0pCiAgfQp9CgojIHJlbW92ZSBhZnRlciBzdW1taW5nIGNvdW50cyAob3RoZXJ3aXNlIElEcyBnZXQgbWl4ZWQgdXApCm5ld0NvdW50cyA8LSBuZXdDb3VudHNbLC10b1JlbW92ZV0KbmV3Q0QgPC0gY2RbLXRvUmVtb3ZlLF0KCiMgQ3JlYXRlIG5ldyBTdW1tYXJpemVkRXhwZXJpbWVudApzZSA8LSBTdW1tYXJpemVkRXhwZXJpbWVudChhc3NheXMgPSBsaXN0KCJjb3VudHMiID0gbmV3Q291bnRzKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIGNvbERhdGEgPSBuZXdDRCwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1ldGFkYXRhID0gbWV0YWRhdGEoc2UxKSkKcm0oc2UxKQpgYGAKCiMgRGF0YSBhbmFseXNpcwoKIyMgQ291bnQgb2JqZWN0CgpgYGB7cn0KZGdlIDwtIERHRUxpc3QoY291bnRzPWFzc2F5KHNlKSkKZGdlJHNhbXBsZQpgYGAKCiMjIERlc2lnbgoKVGhlcmUgY2FuIGJlIGFuIGVmZmVjdCBvZiBhZ2VudCwgdGltZSBpbnRlcmFjdGlvbiBhbmQgYWdlbnQgeCB0aW1lIGludGVyYWN0aW9uLiBXZSBhbHNvIGV4cGVjdCBibG9ja2luZyBmb3IgcGF0aWVudC4gV2UgY2FuIGFzc2VzcyBhbGwgZWZmZWN0cyBvZiBpbnRlcmVzdCB3aXRoaW4gcGF0aWVudC4KCmBgYHtyfQpkZXNpZ24gPC0gbW9kZWwubWF0cml4KH50aW1lKnRyZWF0bWVudCtwYXRpZW50LGNvbERhdGEoc2UpKQpyb3duYW1lcyhkZXNpZ24pID0gY29sbmFtZXMoZGdlKQpkZXNpZ24KCkV4cGxvcmVNb2RlbE1hdHJpeDo6VmlzdWFsaXplRGVzaWduKGNvbERhdGEoc2UpLH4gdGltZSp0cmVhdG1lbnQgKyBwYXRpZW50KSRwbG90bGlzdApgYGAKCiMjIEZpbHRlcmluZwoKYGBge3J9CmtlZXAgPC0gZmlsdGVyQnlFeHByKGRnZSxkZXNpZ24pCnRhYmxlKGtlZXApCmRnZSA8LSBkZ2Vba2VlcCwgLCBrZWVwLmxpYi5zaXplcz1GQUxTRV0KYGBgCgojIyBOb3JtYWxpc2F0aW9uCmBgYHtyfQpkZ2UgPC0gY2FsY05vcm1GYWN0b3JzKGRnZSkKZGdlJHNhbXBsZXMKYGBgCgojIyBEYXRhIGV4cGxvcmF0aW9uCgpBbiBNRFMgcGxvdCBzaG93cyB0aGUgbGVhZGluZyBmb2xkIGNoYW5nZXMgKGRpZmZlcmVudGlhbCBleHByZXNzaW9uKSBiZXR3ZWVuIHRoZSAyMyBzYW1wbGVzLgoKYGBge3J9CnBsb3RNRFMoZGdlLGxhYmVscz1wYXN0ZShjb2xEYXRhKHNlKSR0cmVhdG1lbnQsY29sRGF0YShzZSkkdGltZSxjb2xEYXRhKHNlKSRwYXRpZW50LHNlcD0iLSIpLGNvbD1hcy5kb3VibGUoY29sRGF0YShzZSkkdHJlYXRtZW50KSkKYGBgCgpUaGVyZSBpcyBhIHZlcnkgc3Ryb25nIHBhdGllbnQgZWZmZWN0ISBUbyBmdXJ0aGVyIGFzc2VzcyB0aGUgdHJlYXRtZW50IGVmZmVjdHMgd2UgY2FuIG1ha2UgTURTIHBsb3RzIHBlciBwYXRpZW50CgpgYGB7cn0KZm9yIChpIGluIDE6NCkKcGxvdE1EUyhkZ2VbLGNvbERhdGEoc2UpJHBhdGllbnQ9PWldLCBjb2w9YXMuZG91YmxlKGNvbERhdGEoc2UpJHRyZWF0bWVudClbY29sRGF0YShzZSkkcGF0aWVudD09aV0sCmxhYmVscz1wYXN0ZShjb2xEYXRhKHNlKSR0cmVhdG1lbnRbY29sRGF0YShzZSkkcGF0aWVudD09aV0sCiAgICAgICAgICAgICBjb2xEYXRhKHNlKSR0aW1lW2NvbERhdGEoc2UpJHBhdGllbnQ9PWldLAogICAgICAgICAgICAgY29sRGF0YShzZSkkcGF0aWVudCxzZXA9Ii0iKVtjb2xEYXRhKHNlKSRwYXRpZW50PT1pXSkKYGBgCgoKIyMgUGFyYW1ldGVyIGVzdGltYXRpb24KCldlIHdpbGwgdXNlIHRoZSBkZWZhdWx0IFF1YXNpIGxpa2VsaWhvb2QgYXBwcm9hY2ggb2YgZWRnZVIuCgpGb3IgcXVhc2ktbGlrZWxpaG9vZCB3ZSBkbyBub3Qgc3BlY2lmeSB0aGUgZnVsbCBkaXN0cmlidXRpb24sIG9ubHkgdGhlIGZpcnN0IHR3byBtb21lbnRzOiB0aGUgbWVhbiBhbmQgdGhlIHZhcmlhbmNlLCB3aGljaCBpcyBzdWZmaWNpZW50IHRvIGRvIGluZmVyZW5jZSBvbiB0aGUgbWVhbi4KCiQkClxsZWZ0XHsKXGJlZ2lue2FycmF5fXtsY2x9CkVbeV97aWd9XHZlcnQgXG1hdGhiZnt4fV97aWd9XSY9JlxtdV97aWd9XFwKbG9nKFxtdV97aWd9KSY9JlxldGFfe2lnfVxcClxldGFfe2lnfSY9JlxiZXRhXzAgKyBcYmV0YV97dDJ9IHhfe3QyLGl9ICsgXGJldGFfe0RQTn0geF97RFBOLGl9ICsgXGJldGFfXHRleHR7RFBOOnQyfSB4X3tEUE4saX14X3t0MixpfSBcXCAKJiYgXHF1YWQgKyBcYmV0YV97T0hUfXhfe09IVCxpfSArIFxiZXRhX1x0ZXh0e09IVDp0Mn0geF97T0hULGl9eF97dDIsaX0gXFwgCiYmIFxxdWFkICsgXGJldGFfe3AyfXhfe3AyLGl9ICsgXGJldGFfe3AzfXhfe3AzLGl9ICsgXGJldGFfe3A0fXhfe3A0LGl9XCArIFxsb2cgTl9pXFwKXHRleHR7VmFyfVt5X3tpZ31cdmVydCBcbWF0aGJme3h9X3tpZ31dJj0mXHNpZ21hXjJfZ1xsZWZ0KFxtdV97aWd9K1xwaGlcbXVfe2lnfV4yXHJpZ2h0KQpcZW5ke2FycmF5fVxyaWdodC4KJCQKCndpdGggJFxzaWdtYV4yX2ckIGFuIGFkZGl0aW9uYWwgZGlzcGVyc2lvbiBwYXJhbWV0ZXIgdGhlIHNjYWxlcyB0aGUgbmVnYXRpdmUgYmlub21pYWwgdmFyaWFuY2UgZnVuY3Rpb24sICR4X3tEUE4saX0kLCAkeF97RFBOLGl9JCwgJHhfe3QyLGl9JCwgJHhfe3AuLGl9JCAgZHVtbXkgdmFyaWFibGVzIHRoYXQgaXMgMSBpZiBjZWxsIGxpbmUgd2FzIHRyZWF0ZWQgd2l0aCBEUE4sIE9IVCwgaW5jdWJhdGVkIGZvciA0OCBoLCBmcm9tIHBhdGllbnQgJHAuJCwgcmVzcGVjdGl2ZWx5IGFuZCBpcyAwIG90aGVyd2lzZSwgYW5kLCAkXGxvZ3tOfV9pJCBhIG5vcm1hbGlzYXRpb24gb2Zmc2V0IHRvIGNvcnJlY3QgZm9yIHNlcXVlbmNpbmcgZGVwdGguIE5vdGUsIHRoYXQgJFxiZXRhX3tEUE59JCBpcyB0aGUgbWFpbiBlZmZlY3QgZm9yIHRoZSBEUE4gdHJlYXRtZW50LCBhbmQgY29ycmVzcG9uZHMgdG8gdGhlIGF2ZXJhZ2UgbG9nIGZvbGQgY2hhbmdlIGJldHdlZW4gdHJlYXRlZCBhbmQgY29udHJvbCBtaWNlIGFmdGVyIDI0aC4gVGhlIGludGVyYWN0aW9uICAkXGJldGFfXHRleHR7RFBOOnQyfSQgY2FuIGJlIGludGVycHJldGVkIGFzIHRoZSBhdmVyYWdlIGNoYW5nZSBpbiBsb2cyIEZDIGJldHdlZW4gRFBOIHRyZWF0ZWQgYW5kIGNvbnRyb2wgY2VsbCBsaW5lcyBhdCB0aGUgbGF0ZSBhbmQgZWFybHkgdGltZXBvaW50LiBUaGUgcmVzZWFyY2hlcnMgYXJlIGFsc28gaW50ZXJlc3RlZCBpbiBhIGFzc2Vzc2luZyB0aGlyZCBjb250cmFzdDogdGhlIGVmZmVjdCBvZiB0aGUgRFBOIHRyZWF0bWVudCBhdCB0aGUgbGF0ZSB0aW1lIHBvaW50LgoKJCQgXGxvZ18yXHRleHR7RkN9Xlx0ZXh0ezQ4aH1fXHRleHR7RFBOIC0gQ309IFxiZXRhX3tEUE59K1xiZXRhX3tEUE4sdDJ9JCQKCkZvciB0aGUgT0hUIHRyZWF0bWVudCB3ZSB3aWxsIGFzc2VzcyBzaW1pbGFyIGNvbnRyYXN0cy4gCgokJCBcbG9nXzJcdGV4dHtGQ31eXHRleHR7MjRofV9cdGV4dHtPSFQgLSBDfT0gXGJldGFfe09IVH0kJAokJCBcbG9nXzJcdGV4dHtGQ31eXHRleHR7NDhofV9cdGV4dHtPSFQgLSBDfT0gXGJldGFfe09IVH0rXGJldGFfe09IVCx0Mn0kJAokJCBcbG9nXzJcdGV4dHtGQ31eXHRleHR7NDhofV9cdGV4dHtPSFQgLSBDfSAtXGxvZ18yXHRleHR7RkN9Xlx0ZXh0ezI0aH1fXHRleHR7T0hUIC0gQ309IFxiZXRhX3tPSFQsdDJ9CiQkCgpGaW5hbGx5LCB3ZSBhbHNvIGhhdmUgdG8gYXNzZXNzIGlmIHRoZXJlIGlzIGEgZGlmZmVyZW5jZSBiZXR3ZWVuIERQTiBhbmQgT0hUIHRyZWF0bWVudAoKJCQgXGxvZ18yXHRleHR7RkN9Xlx0ZXh0ezI0aH1fXHRleHR7T0hUIC0gRFBOfT0gXGJldGFfe09IVH0gLSBcYmV0YV97RFBOfSQkCiQkIFxsb2dfMlx0ZXh0e0ZDfV5cdGV4dHs0OGh9X1x0ZXh0e09IVCAtIEN9PSBcYmV0YV97T0hUfStcYmV0YV97T0hULHQyfSAtIFxiZXRhX3tEUE59IC0gXGJldGFfe0RQTix0Mn0KJCQKCgokJCBcbG9nXzJcdGV4dHtGQ31eXHRleHR7NDhofV9cdGV4dHtPSFQgLSBEUE59IC1cbG9nXzJcdGV4dHtGQ31eXHRleHR7MjRofV9cdGV4dHtPSFQgLSBEUE59PSBcYmV0YV97T0hULHQyfSAtIFxiZXRhX3tEUE4sdDJ9CiQkCgoKCgoKCmBgYHtyfQpkZ2UgPC0gZXN0aW1hdGVEaXNwKGRnZSwgZGVzaWduKSAgCnBsb3RCQ1YoZGdlKQpgYGAKClRoZSBxdWFzaS1uZWdhdGl2ZSBiaW5vbWlhbCBtb2RlbCBjYW4gYmUgZml0dGVkIHVzaW5nIHRoZSBmdW5jdGlvbiBgZ2xtUUxGaXRgCmBgYHtyfQpmaXQgPC0gZ2xtUUxGaXQoZGdlLGRlc2lnbikKYGBgCgoKIyMgQ29udHJhc3RzCgpXZSBub3cgaW1wbGVtZW50IGFsbCA5IGNvbnRyYXN0cyBvZiBpbnRlcmVzdCAKCmBgYHtyfQpMIDwtIG1zcXJvYjI6Om1ha2VDb250cmFzdCgKICAgIGMoInRyZWF0bWVudERQTiA9IDAiLAogICAgICAidHJlYXRtZW50T0hUID0gMCIsCiAgICAgICJ0cmVhdG1lbnRPSFQgLSB0cmVhdG1lbnREUE4gPSAwIiwgCiAgICAgICJ0cmVhdG1lbnREUE4gKyB0aW1lNDhoOnRyZWF0bWVudERQTiA9IDAiLAogICAgICAidHJlYXRtZW50T0hUICsgdGltZTQ4aDp0cmVhdG1lbnRPSFQgPSAwIiwKICAgICAgInRyZWF0bWVudE9IVCArIHRpbWU0OGg6dHJlYXRtZW50T0hUIC0gdHJlYXRtZW50RFBOIC0gdGltZTQ4aDp0cmVhdG1lbnREUE4gPSAwIiwgCiAgICAgICJ0aW1lNDhoOnRyZWF0bWVudERQTiA9IDAiLCAKICAgICAgInRpbWU0OGg6dHJlYXRtZW50T0hUID0gMCIsIAogICAgICAidGltZTQ4aDp0cmVhdG1lbnRPSFQgLSB0aW1lNDhoOnRyZWF0bWVudERQTiA9IDAiKSwKICAgIHBhcmFtZXRlck5hbWVzID0gY29sbmFtZXMoZGVzaWduKSkKTApgYGAKCiMjIFRlc3RzCgpXZSBoYXZlIHRvIHBlcmZvcm0gYSBxdWFzaS0gRi10ZXN0IGZvciBlYWNoIGNvbnRyYXN0LiAKVGhlIHF1YXNpIEYtdGVzdCBpbnZvbHZlcyBmaXR0aW5nIGEgZGlmZmVyZW50IG1vZGVsIGZvciBlYWNoIGNvbnRyYXN0IHNvIHRoYXQgd2UgY2FuIGNvbXBhcmUgdGhlIGZ1bGwgbW9kZWwgd2l0aCBhIHJlZHVjZWQgbW9kZWwgdGhhdCBpbXBsaWVzIHRoYXQgb25lIHNwZWNpZmljIGNvbnRyYXN0IGlzIHplcm8uIAoKQmVjYXVzZSB3ZSBlc3RpbWF0ZWQgdGhlIGFkZGl0aW9uYWwgZGlzcGVyc2lvbiBwYXJhbWV0ZXIgJFxzaWdtYV4yX2ckIHVzaW5nIGEgc3VtIG9mIHNxdWFyZWQgZGV2aWFuY2UgcmVzaWR1YWxzOiAKCmkuZS4gCiQkZV97aSxkfSA9IDIgKGxfaSh5X2kseV9pKSAtIGxfaShcbXVfaSx5X2kpJCQgCmFuZCAKCiQkIApcaGF0IFxzaWdtYV9nXjIgPSBcZnJhY3tcc3VtX3tpPTF9Xm4gZV97aSxkfV4yfXtuLXB9ID0gXGZyYWN7MlxsZWZ0W2woXG1hdGhiZnt5fSxcbWF0aGJme3l9KSAtIGwoXGJvbGRzeW1ib2x7XG11fSxcbWF0aGJme3l9KVxyaWdodF19e24tcH0KJCQKCldpdGggZWRnZVIgd2Ugd2lsbCB0aGVuIGZ1cnRoZXIgYWRvcHQgZW1waXJpY2FsIEJheWVzIHRvIGJvcnJvdyBzdHJlbmd0aCBhY3Jvc3MgZ2VuZXMgdG8gc3RhYmlsaXNlIHRoZSBwYXJhbWV0ZXIgZXN0aW1hdG9yLCB3aGljaCB3aWxsIGFsc28gaW5jcmVhc2UgdGhlIGRlZ3JlZXMgaWYgdGhpcyBnZW5lIHdpc2UgZGlzcGVyc2lvbiBwYXJhbWV0ZXIgZXN0aW1hdG9yIHdoaWNoIHdlIHJlZmVyIHRvIGFzICRkZl9cdGV4dHtyZXN9XntFQn0kLgoKV2UgY2FuIHVzZSBhIHF1YXNpIEYgLXRlc3QgdGhhdCBjYW4gYWxzbyBjb3JyZWN0IGZvciB0aGUgZGVncmVlcyBvZiBmcmVlZG9tIHRoYXQgaGF2ZSBiZWVuIHVzZWQgdG8gZXN0aW1hdGUgbWVhbiBtb2RlbCBwYXJhbWV0ZXJzIGFuZCB0aGUgcmVzaWR1YWwgZGVncmVlcyBvZiBmcmVlZG9tIHRoYXQgd2VyZSBhdmFpbGFibGUgZm9yIGVzdGltYXRpbmcgdGhlIGFkZGl0aW9uYWwgZGlzcGVyc2lvbiBwYXJhbWV0ZXIuIFRoZSBxdWFzaSBGIHRlc3Qgd2lsbCB0aHVzIHBlcmZvcm0gYmV0dGVyIGluIGEgc21hbGwgc2FtcGxlIHNldHRpbmcuIEl0IGlzIGRlZmluZWQgYXM6IAoKJCQKRiA9IFxmcmFje1xmcmFje0xSVF9cdGV4dHtnLCBmdWxsIC0gcmVkdWNlZH19e2RmX3tMUlR9fX17XHNpZ21hXjJfZ30gCiQkCgp3aGljaCBmb2xsb3dzIGFuIEYgLSBkaXN0cmlidXRpb24gd2l0aCAkZGZfe0xSVH0kIGRlZ3JlZXMgb2YgZnJlZWRvbSBpbiB0aGUgbm9taW5hdG9yIGFuZCAkZGZfe3Jlc31ee0VCfSQgZGVncmVlcyBvZiBmcmVlZG9tIGluIHRoZSBkZW5vbWluYXRvciB1bmRlciB0aGUgbnVsbCBoeXBvdGhlc2lzIHRoYXQgdGhlIGZ1bGwgYW5kIHJlZHVjZWQgbW9kZWwgYXJlIGVxdWl2YWxlbnQgYW5kIHRoYXQgdGhlIGFzc2Vzc2VkIGNvbnRyYXN0cyBhcmUgdGh1cyBlcXVhbCB0byB6ZXJvLiBJbmRlZWQsIHRoZSBkaXNwZXJzaW9uIGVzdGltYXRvciBpbiB0aGUgZGVub21pbmF0b3IgZm9sbG93cyBhIHNjYWxlZCAkXGNoaV4yJCBkaXN0cmlidXRpb24gd2l0aCAkZGZfe3Jlc31ee0VCfSQgZGVncmVlcyBvZiBmcmVlZG9tLiAKCldlIHBlcmZvcm0gYWxsIHRlc3RzIGFuZCBsb29wIG92ZXIgdGhlIGNvbHVtbnMgb2YgTCBmb3IgdGhpcyBwdXJwb3NlLiAKCmBgYHtyfQp0ZXN0c0YgPC0gYXBwbHkoTCwgMiwgZnVuY3Rpb24oZml0LGNvbnRyYXN0KSAKICAgIGdsbVFMRlRlc3QoZml0LGNvbnRyYXN0PWNvbnRyYXN0KSwgCiAgICBmaXQgPSBmaXQpCnRvcFRhYmxlc0Y8LSBsYXBwbHkodGVzdHNGLCB0b3BUYWdzLCBuPW5yb3coZGdlKSkKCnNhcHBseSh0b3BUYWJsZXNGLCBmdW5jdGlvbih4KSBzdW0oeCR0YWJsZSRGRFIgPDAuMDUpKQpgYGAKCldlIG9ubHkgZmluZCBzaWduaWZpY2FudCBmb2xkIGNoYW5nZXMgZm9yIHZlcnkgZmV3IGdlbmVzIGJldHdlZW4gT0hUIGFuZCB0aGUgY29udHJvbCB0cmVhdG1lbnQgYXQgdGhlIGxhdGUgdGltZSBwb2ludC4gCldlIGFsc28gZGlkIG5vdCBmaW5kIHNpZ25pZmljYW50IGludGVyYWN0aW9ucy4gCgojIFBsb3RzIAojIyBWb2xjYW5vIHBsb3RzIAoKYGBge3J9CmZvciAoaSBpbiAxOm5jb2woTCkpCnsKICAgdm9sY2FubzwtIGdncGxvdCh0b3BUYWJsZXNGW1tpXV0kdGFibGUsYWVzKHg9bG9nRkMseT0tbG9nMTAoUFZhbHVlKSxjb2xvcj1GRFIgPCAwLjA1KSkgKyBnZW9tX3BvaW50KCkgKyBzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzPWMoImJsYWNrIiwicmVkIikpICsgZ2d0aXRsZShwYXN0ZSgiY29udHJhc3QiLG5hbWVzKHRvcFRhYmxlc0YpW2ldKSkKcHJpbnQodm9sY2FubykKfQpgYGAKCiMjIEhpc3RvZ3JhbXMgb2YgcC12YWx1ZXMKCmBgYHtyfQpoaXN0c1AgPC0gbGFwcGx5KHRvcFRhYmxlc0YsIGZ1bmN0aW9uKHgpIAogICAgeCR0YWJsZSAlPiUgCiAgICAgICAgZ2dwbG90KGFlcyh4PVBWYWx1ZSkpICsgCiAgICAgICAgZ2VvbV9oaXN0b2dyYW0oYnJlYWtzID1zZXEoMCwxLC4xKSAsY29sPTEpCiAgICApCiAgICAKZm9yIChpIGluIDE6bmNvbChMKSkgCiAgICBoaXN0c1BbW2ldXSA8LSBoaXN0c1BbW2ldXSArCiAgICBnZ3RpdGxlKHBhc3RlKCJjb250cmFzdCIsbmFtZXModG9wVGFibGVzRilbaV0pKQpoaXN0c1AKYGBgCgojIyBoZWF0bWFwcwoKYGBge3J9CmZvciAoaSBpbiAxOm5jb2woTCkpCnsKICAgIHNpZ0lEIDwtIHRvcFRhYmxlc0ZbW2ldXSR0YWJsZSAlPiUgCiAgICAgICAgZmlsdGVyKEZEUjwwLjA1KSAlPiUgCiAgICAgICAgcm93bmFtZXMKICAgIGlmIChsZW5ndGgoc2lnSUQpPjApCiAgICAgICAgaGVhdG1hcChkZ2UkY291bnRzW3NpZ0lELF0sIG1haW4gPSBjb2xuYW1lcyhMKVtpXSwgY2V4Lm1haW49LjIpCn0KYGBgCgojIEVkZ2VSIHRyYWRpdGlvbmFsIAoKYGBge3J9CmZpdEdsbSA8LSBnbG1GaXQoZGdlLGRlc2lnbikKdGVzdExSVDIgPC0gYXBwbHkoTCwgMiwgZnVuY3Rpb24oZml0LGNvbnRyYXN0KSAKICAgIGdsbUxSVChmaXQsY29udHJhc3Q9Y29udHJhc3QpLCAKICAgIGZpdCA9IGZpdEdsbSkKdG9wVGFibGVzTFJUMiA8LSBsYXBwbHkodGVzdExSVDIsIHRvcFRhZ3MsIG49bnJvdyhkZ2UpKQpzYXBwbHkodG9wVGFibGVzTFJUMiwgZnVuY3Rpb24oeCkgc3VtKHgkdGFibGUkRkRSIDwwLjA1KSkKYGBgCgpXZSBmaW5kIG1vcmUgZ2VuZXMgZm9yIHRoZSB0cmFkaXRpb25hbCBlZGdlUiB3b3JrZmxvdywgaG93ZXZlciwgaXQgaXMga25vd24gdGhhdCB0aGlzIHdvcmtmbG93IGlzIG9mdGVuIHRvbyBsaWJlcmFsLiAK