# Using DelayedMatrix with MultiAssayExperiment

MultiAssay Special Interest Group

#### October 26, 2021

# Contents

# 1 Integrating an HDF5 backend for MultiAssayExperiment

## 1.1 Dependencies

    library(MultiAssayExperiment)
    library(HDF5Array)
    library(SummarizedExperiment)

## 1.2 HDF5Array and DelayedArray Constructor

The `HDF5Array` package provides an on-disk representation of large datasets without the need to load them into memory. Convenient lazy evaluation operations allow the user to manipulate such large data files based on metadata. The `DelayedMatrix` class in the `DelayedArray` package provides a way to connect to a large matrix that is stored on disk.

First, we create a small matrix for constructing the `DelayedMatrix` class.

    smallMatrix <- matrix(rnorm(10e5), ncol = 20)

We add rownames and column names to the matrix object for compatibility with the `MultiAssayExperiment` representation.

    rownames(smallMatrix) <- paste0("GENE", seq_len(nrow(smallMatrix)))
    colnames(smallMatrix) <- paste0("SampleID", seq_len(ncol(smallMatrix)))

Here we use the `DelayedArray` constructor function to create a `DelayedMatrix` object.

    smallMatrix <- DelayedArray(smallMatrix)
    class(smallMatrix)

    ## [1] "DelayedMatrix"
    ## attr(,"package")
    ## [1] "DelayedArray"

    # show method
    smallMatrix

    ## <50000 x 20> matrix of class DelayedMatrix and type "double":
    ##             SampleID1   SampleID2   SampleID3 ...  SampleID19  SampleID20
    ##     GENE1  0.16582313  0.30709133  0.32523368   .  0.38840105 -0.30037180
    ##     GENE2  1.43362884 -2.19596856  2.00637017   .  0.71849913  0.12676298
    ##     GENE3 -0.86792995  0.14822969 -0.76133267   .  0.90869337  0.23863274
    ##     GENE4 -0.40733134  0.02297488 -0.11342149   .  0.07734068 -0.23571734
    ##     GENE5 -0.39897510 -0.09999889  1.08147850   .  0.06753737  0.92162908
    ##       ...           .           .           .   .           .           .
    ## GENE49996  -0.2512429  -0.2844520  -0.2742080   .  0.47893378 -0.44982823
    ## GENE49997  -1.2690389   1.1677833   0.6704477   . -0.27967909 -0.38797136
    ## GENE49998  -2.5460499   0.9136259   0.4110660   .  1.87833136  0.42925015
    ## GENE49999  -0.1939874   0.6584898  -0.3627206   . -0.07650611  0.25406255
    ## GENE50000   0.1037659  -0.1727839  -0.1992474   .  0.75040693 -0.57068756

    dim(smallMatrix)

    ## [1] 50000    20

## 1.3 Writing to a file with dimnames

Finally, the `rhdf5` package stores `dimnames` in a standard location.

In order to make use of this functionality, we would use `writeHDF5Array` with the `with.dimnames` argument:

    testh5 <- tempfile(fileext = ".h5")
    writeHDF5Array(smallMatrix, filepath = testh5, name = "smallMatrix",
        with.dimnames = TRUE)

    ## <50000 x 20> matrix of class HDF5Matrix and type "double":
    ##             SampleID1   SampleID2   SampleID3 ...  SampleID19  SampleID20
    ##     GENE1  0.16582313  0.30709133  0.32523368   .  0.38840105 -0.30037180
    ##     GENE2  1.43362884 -2.19596856  2.00637017   .  0.71849913  0.12676298
    ##     GENE3 -0.86792995  0.14822969 -0.76133267   .  0.90869337  0.23863274
    ##     GENE4 -0.40733134  0.02297488 -0.11342149   .  0.07734068 -0.23571734
    ##     GENE5 -0.39897510 -0.09999889  1.08147850   .  0.06753737  0.92162908
    ##       ...           .           .           .   .           .           .
    ## GENE49996  -0.2512429  -0.2844520  -0.2742080   .  0.47893378 -0.44982823
    ## GENE49997  -1.2690389   1.1677833   0.6704477   . -0.27967909 -0.38797136
    ## GENE49998  -2.5460499   0.9136259   0.4110660   .  1.87833136  0.42925015
    ## GENE49999  -0.1939874   0.6584898  -0.3627206   . -0.07650611  0.25406255
    ## GENE50000   0.1037659  -0.1727839  -0.1992474   .  0.75040693 -0.57068756

To see the file structure we use `h5ls`:

    h5ls(testh5)

    ##                    group                  name       otype dclass        dim
    ## 0                      / .smallMatrix_dimnames   H5I_GROUP
    ## 1 /.smallMatrix_dimnames                     1 H5I_DATASET STRING      50000
    ## 2 /.smallMatrix_dimnames                     2 H5I_DATASET STRING         20
    ## 3                      /           smallMatrix H5I_DATASET  FLOAT 50000 x 20

## 1.4 Importing HDF5 files

Note that a large matrix from an HDF5 file can also be loaded using the `HDF5ArraySeed` and `DelayedArray` functions.

    hdf5Data <- HDF5ArraySeed(file = testh5, name = "smallMatrix")
    newDelayedMatrix <- DelayedArray(hdf5Data)
    class(newDelayedMatrix)

    ## [1] "HDF5Matrix"
    ## attr(,"package")
    ## [1] "HDF5Array"

    newDelayedMatrix

    ## <50000 x 20> matrix of class HDF5Matrix and type "double":
    ##             SampleID1   SampleID2   SampleID3 ...  SampleID19  SampleID20
    ##     GENE1  0.16582313  0.30709133  0.32523368   .  0.38840105 -0.30037180
    ##     GENE2  1.43362884 -2.19596856  2.00637017   .  0.71849913  0.12676298
    ##     GENE3 -0.86792995  0.14822969 -0.76133267   .  0.90869337  0.23863274
    ##     GENE4 -0.40733134  0.02297488 -0.11342149   .  0.07734068 -0.23571734
    ##     GENE5 -0.39897510 -0.09999889  1.08147850   .  0.06753737  0.92162908
    ##       ...           .           .           .   .           .           .
    ## GENE49996  -0.2512429  -0.2844520  -0.2742080   .  0.47893378 -0.44982823
    ## GENE49997  -1.2690389   1.1677833   0.6704477   . -0.27967909 -0.38797136
    ## GENE49998  -2.5460499   0.9136259   0.4110660   .  1.87833136  0.42925015
    ## GENE49999  -0.1939874   0.6584898  -0.3627206   . -0.07650611  0.25406255
    ## GENE50000   0.1037659  -0.1727839  -0.1992474   .  0.75040693 -0.57068756

## 1.5 Using a `DelayedMatrix` with `MultiAssayExperiment`

A `DelayedMatrix` alone conforms to the `MultiAssayExperiment` API requirements. Shown below, the `DelayedMatrix` can be put into a named `list` and passed into the `MultiAssayExperiment` constructor function.

    HDF5MAE <- MultiAssayExperiment(experiments = list(smallMatrix = smallMatrix))
    sampleMap(HDF5MAE)

    ## DataFrame with 20 rows and 3 columns
    ##           assay     primary     colname
    ##        <factor> <character> <character>
    ## 1   smallMatrix   SampleID1   SampleID1
    ## 2   smallMatrix   SampleID2   SampleID2
    ## 3   smallMatrix   SampleID3   SampleID3
    ## 4   smallMatrix   SampleID4   SampleID4
    ## 5   smallMatrix   SampleID5   SampleID5
    ## ...         ...         ...         ...
    ## 16  smallMatrix  SampleID16  SampleID16
    ## 17  smallMatrix  SampleID17  SampleID17
    ## 18  smallMatrix  SampleID18  SampleID18
    ## 19  smallMatrix  SampleID19  SampleID19
    ## 20  smallMatrix  SampleID20  SampleID20

    colData(HDF5MAE)

    ## DataFrame with 20 rows and 0 columns

### 1.5.1 `SummarizedExperiment` with `DelayedMatrix` backend

A more information rich `DelayedMatrix` can be created when used in conjunction with the `SummarizedExperiment` class and it can even include `rowRanges`. The flexibility of the `MultiAssayExperiment` API supports classes with minimal requirements. Additionally, this `SummarizedExperiment` with the `DelayedMatrix` backend can be part of a bigger `MultiAssayExperiment` object. Below is a minimal example of how this would work:

    HDF5SE <- SummarizedExperiment(assays = smallMatrix)
    assay(HDF5SE)

    ## <50000 x 20> matrix of class DelayedMatrix and type "double":
    ##             SampleID1   SampleID2   SampleID3 ...  SampleID19  SampleID20
    ##     GENE1  0.16582313  0.30709133  0.32523368   .  0.38840105 -0.30037180
    ##     GENE2  1.43362884 -2.19596856  2.00637017   .  0.71849913  0.12676298
    ##     GENE3 -0.86792995  0.14822969 -0.76133267   .  0.90869337  0.23863274
    ##     GENE4 -0.40733134  0.02297488 -0.11342149   .  0.07734068 -0.23571734
    ##     GENE5 -0.39897510 -0.09999889  1.08147850   .  0.06753737  0.92162908
    ##       ...           .           .           .   .           .           .
    ## GENE49996  -0.2512429  -0.2844520  -0.2742080   .  0.47893378 -0.44982823
    ## GENE49997  -1.2690389   1.1677833   0.6704477   . -0.27967909 -0.38797136
    ## GENE49998  -2.5460499   0.9136259   0.4110660   .  1.87833136  0.42925015
    ## GENE49999  -0.1939874   0.6584898  -0.3627206   . -0.07650611  0.25406255
    ## GENE50000   0.1037659  -0.1727839  -0.1992474   .  0.75040693 -0.57068756

    MultiAssayExperiment(list(HDF5SE = HDF5SE))

    ## A MultiAssayExperiment object of 1 listed
    ##  experiment with a user-defined name and respective class.
    ##  Containing an ExperimentList class object of length 1:
    ##  [1] HDF5SE: SummarizedExperiment with 50000 rows and 20 columns
    ## Functionality:
    ##  experiments() - obtain the ExperimentList instance
    ##  colData() - the primary/phenotype DataFrame
    ##  sampleMap() - the sample coordination DataFrame
    ##  `$`, `[`, `[[` - extract colData columns, subset, or experiment
    ##  *Format() - convert into a long or wide DataFrame
    ##  assays() - convert ExperimentList to a SimpleList of matrices
    ##  exportClass() - save data to flat files

Additional scenarios are currently in development where an `HDF5Matrix` is hosted remotely. Many opportunities exist when considering on-disk and off-disk representations of data with `MultiAssayExperiment`.