cg
changeset 8:3bc61ab8e776
.
author | bshanks@bshanks.dyndns.org |
---|---|
date | Sat Apr 11 21:31:27 2009 -0700 (16 years ago) |
parents | 075618f574d8 |
children | 3480ab8239f5 |
files | DocumentConverter.py grant.html grant.odt grant.txt |
line diff
1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/DocumentConverter.py Sat Apr 11 21:31:27 2009 -0700
1.3 @@ -0,0 +1,151 @@
1.4 +#!/usr/bin/python
1.5 +#
1.6 +# PyODConverter (Python OpenDocument Converter) v1.0.0 - 2008-05-05
1.7 +#
1.8 +# This script converts a document from one office format to another by
1.9 +# connecting to an OpenOffice.org instance via Python-UNO bridge.
1.10 +#
1.11 +# Copyright (C) 2008 Mirko Nasato <mirko@artofsolving.com>
1.12 +# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl-2.1.html
1.13 +# - or any later version.
1.14 +#
1.15 +DEFAULT_OPENOFFICE_PORT = 8100
1.16 +
1.17 +import uno
1.18 +from os.path import abspath, isfile, splitext
1.19 +from com.sun.star.beans import PropertyValue
1.20 +from com.sun.star.task import ErrorCodeIOException
1.21 +from com.sun.star.connection import NoConnectException
1.22 +
1.23 +FAMILY_TEXT = "Text"
1.24 +FAMILY_SPREADSHEET = "Spreadsheet"
1.25 +FAMILY_PRESENTATION = "Presentation"
1.26 +FAMILY_DRAWING = "Drawing"
1.27 +
1.28 +FILTER_MAP = {
1.29 + "pdf": {
1.30 + FAMILY_TEXT: "writer_pdf_Export",
1.31 + FAMILY_SPREADSHEET: "calc_pdf_Export",
1.32 + FAMILY_PRESENTATION: "impress_pdf_Export",
1.33 + FAMILY_DRAWING: "draw_pdf_Export"
1.34 + },
1.35 + "html": {
1.36 + FAMILY_TEXT: "HTML (StarWriter)",
1.37 + FAMILY_SPREADSHEET: "HTML (StarCalc)",
1.38 + FAMILY_PRESENTATION: "impress_html_Export"
1.39 + },
1.40 + "odt": { FAMILY_TEXT: "writer8" },
1.41 + "doc": { FAMILY_TEXT: "MS Word 97" },
1.42 + "rtf": { FAMILY_TEXT: "Rich Text Format" },
1.43 + "txt": { FAMILY_TEXT: "Text" },
1.44 + "ods": { FAMILY_SPREADSHEET: "calc8" },
1.45 + "xls": { FAMILY_SPREADSHEET: "MS Excel 97" },
1.46 + "odp": { FAMILY_PRESENTATION: "impress8" },
1.47 + "ppt": { FAMILY_PRESENTATION: "MS PowerPoint 97" },
1.48 + "swf": { FAMILY_PRESENTATION: "impress_flash_Export" }
1.49 +}
1.50 +# see http://wiki.services.openoffice.org/wiki/Framework/Article/Filter
1.51 +# for more available filters
1.52 +
1.53 +
1.54 +class DocumentConversionException(Exception):
1.55 +
1.56 + def __init__(self, message):
1.57 + self.message = message
1.58 +
1.59 + def __str__(self):
1.60 + return self.message
1.61 +
1.62 +
1.63 +class DocumentConverter:
1.64 +
1.65 + def __init__(self, port=DEFAULT_OPENOFFICE_PORT):
1.66 + localContext = uno.getComponentContext()
1.67 + resolver = localContext.ServiceManager.createInstanceWithContext("com.sun.star.bridge.UnoUrlResolver", localContext)
1.68 + try:
1.69 + context = resolver.resolve("uno:socket,host=localhost,port=%s;urp;StarOffice.ComponentContext" % port)
1.70 + except NoConnectException:
1.71 + raise DocumentConversionException, "failed to connect to OpenOffice.org on port %s" % port
1.72 + self.desktop = context.ServiceManager.createInstanceWithContext("com.sun.star.frame.Desktop", context)
1.73 +
1.74 + def convert(self, inputFile, outputFile):
1.75 +
1.76 + inputUrl = self._toFileUrl(inputFile)
1.77 + outputUrl = self._toFileUrl(outputFile)
1.78 +
1.79 + document = self.desktop.loadComponentFromURL(inputUrl, "_blank", 0, self._toProperties(Hidden=True))
1.80 + try:
1.81 + document.refresh()
1.82 + except AttributeError:
1.83 + pass
1.84 +
1.85 + outputExt = self._getFileExt(outputFile)
1.86 + filterName = self._filterName(document, outputExt)
1.87 +
1.88 + try:
1.89 + document.storeToURL(outputUrl, self._toProperties(FilterName=filterName))
1.90 + finally:
1.91 + document.close(True)
1.92 +
1.93 + def _filterName(self, document, outputExt):
1.94 + family = self._detectFamily(document)
1.95 + try:
1.96 + filterByFamily = FILTER_MAP[outputExt]
1.97 + except KeyError:
1.98 + raise DocumentConversionException, "unknown output format: '%s'" % outputExt
1.99 + try:
1.100 + return filterByFamily[family]
1.101 + except KeyError:
1.102 + raise DocumentConversionException, "unsupported conversion: from '%s' to '%s'" % (family, outputExt)
1.103 +
1.104 + def _detectFamily(self, document):
1.105 + if document.supportsService("com.sun.star.text.GenericTextDocument"):
1.106 + # NOTE: a GenericTextDocument is either a TextDocument, a WebDocument, or a GlobalDocument
1.107 + # but this further distinction doesn't seem to matter for conversions
1.108 + return FAMILY_TEXT
1.109 + if document.supportsService("com.sun.star.sheet.SpreadsheetDocument"):
1.110 + return FAMILY_SPREADSHEET
1.111 + if document.supportsService("com.sun.star.presentation.PresentationDocument"):
1.112 + return FAMILY_PRESENTATION
1.113 + if document.supportsService("com.sun.star.drawing.DrawingDocument"):
1.114 + return FAMILY_DRAWING
1.115 + raise DocumentConversionException, "unknown document family: %s" % document
1.116 +
1.117 + def _getFileExt(self, path):
1.118 + ext = splitext(path)[1]
1.119 + if ext is not None:
1.120 + return ext[1:].lower()
1.121 +
1.122 + def _toFileUrl(self, path):
1.123 + return uno.systemPathToFileUrl(abspath(path))
1.124 +
1.125 + def _toProperties(self, **args):
1.126 + props = []
1.127 + for key in args:
1.128 + prop = PropertyValue()
1.129 + prop.Name = key
1.130 + prop.Value = args[key]
1.131 + props.append(prop)
1.132 + return tuple(props)
1.133 +
1.134 +
1.135 +if __name__ == "__main__":
1.136 + from sys import argv, exit
1.137 +
1.138 + if len(argv) < 3:
1.139 + print "USAGE: python %s <input-file> <output-file>" % argv[0]
1.140 + exit(255)
1.141 + if not isfile(argv[1]):
1.142 + print "no such input file: %s" % argv[1]
1.143 + exit(1)
1.144 +
1.145 + try:
1.146 + converter = DocumentConverter()
1.147 + converter.convert(argv[1], argv[2])
1.148 + except DocumentConversionException, exception:
1.149 + print "ERROR!" + str(exception)
1.150 + exit(1)
1.151 + except ErrorCodeIOException, exception:
1.152 + print "ERROR! ErrorCodeIOException %d" % exception.ErrCode
1.153 + exit(1)
1.154 +
2.1 --- a/grant.html Sat Apr 11 19:59:01 2009 -0700
2.2 +++ b/grant.html Sat Apr 11 21:31:27 2009 -0700
2.3 @@ -1,5 +1,6 @@
2.4 Specific aims
2.5 - Massive new datasets obtained with techniques such as in situ hybridization
2.6 + todo test3
2.7 + Massive new datasets obtained with techniques such as in situ hybridization
2.8 (ISH) and BAC-transgenics allow the expression levels of many genes at many
2.9 locations to be compared. Our goal is to develop automated methods to relate
2.10 spatial variation in gene expression to anatomy. We want to find marker genes
2.11 @@ -34,10 +35,10 @@
2.12 determine to which subregion each voxel within the structure belongs. We call
2.13 this a classification task, because each voxel is being assigned to a class (namely,
2.14 its subregion).
2.15 + 1
2.16 +
2.17 Therefore, an understanding of the relationship between the combination of
2.18 their expression levels and the locations of the subregions may be expressed as
2.19 - 1
2.20 -
2.21 a function. The input to this function is a voxel, along with the gene expression
2.22 levels within that voxel; the output is the subregional identity of the target
2.23 voxel, that is, the subregion to which the target voxel belongs. We call this
2.24 @@ -79,11 +80,11 @@
2.25 Key questions when choosing a learning method are: What are the instances?
2.26 What are the features? How are the features chosen? Here are four principles
2.27 that outline our answers to these questions.
2.28 + 2
2.29 +
2.30 Principle 1: Combinatorial gene expression
2.31 Above, we defined an “instance” as the combination of a voxel with the “asso-
2.32 ciated gene expression data”. In our case this refers to the expression level of
2.33 - 2
2.34 -
2.35 genes within the voxel, but should we include the expression levels of all genes,
2.36 or only a few of them?
2.37 It is too much to hope that every anatomical region of interest will be iden-
2.38 @@ -121,10 +122,10 @@
2.39 the analysis algorithm to take advantage of this prior knowledge. In addition,
2.40 it is easier for humans to visualize and work with 2-D data.
2.41 Therefore, when possible, the instances should represent pixels, not voxels.
2.42 + 3
2.43 +
2.44 Aim 2
2.45 todo
2.46 - 3
2.47 -
2.48 Aim 3
2.49 Background
2.50 The cortex is divided into areas and layers. To a first approximation, the par-
2.51 @@ -164,11 +165,11 @@
2.52 day cortical maps was driven by the application of histological stains. It is
2.53 conceivable that if a different set of stains had been available which identified
2.54 a different set of features, then the today’s cortical maps would have come out
2.55 + 4
2.56 +
2.57 differently. Since the number of classes of stains is small compared to the number
2.58 of genes, it is likely that there are many repeated, salient spatial patterns in
2.59 the gene expression which have not yet been captured by any stain. Therefore,
2.60 - 4
2.61 -
2.62 current ideas about cortical anatomy need to incorporate what we can learn
2.63 from looking at the patterns of gene expression.
2.64 While we do not here propose to analyze human gene expression data, it is
2.65 @@ -199,9 +200,7 @@
2.66 expression profiles. We achieved classification accuracy of about 81%3. As noted
2.67 above, however, a classifier that looks at all the genes at once isn’t practically
2.68 useful.
2.69 - The requirement to find combinations of only a small number of genes limits
2.70 - us from straightforwardly applying many of the most simple techniques from
2.71 -__________________________
2.72 +_____________________
2.73 1“WW, C2 and coiled-coil domain containing 1”; EntrezGene ID 211652
2.74 2“mitochondrial translational initiation factor 2”; EntrezGene ID 76784
2.75 3Using the Shogun SVM package (todo:cite), with parameters type=GMNPSVM (multi-
2.76 @@ -222,13 +221,8 @@
2.77 expression and blue meaning little.
2.78 6
2.79
2.80 -
2.81 -
2.82 - Figure 2: The top row shows the three genes which (individually) best predict
2.83 - area AUD, according to logistic regression. The bottom row shows the three
2.84 - genes which (individually) best match area AUD, according to gradient similar-
2.85 - ity. From left to right and top to bottom, the genes are Ssr1, Efcbp1, Aph1a,
2.86 - Ptk7, Aph1a again, and Lepr
2.87 + The requirement to find combinations of only a small number of genes limits
2.88 + us from straightforwardly applying many of the most simple techniques from
2.89 the field of supervised machine learning. In the parlance of machine learning,
2.90 our task combines feature selection with supervised learning.
2.91 Principle 3: Use geometry
2.92 @@ -246,16 +240,6 @@
2.93 may be particularly good markers. None of these genes are, individually, a
2.94 perfect marker for AUD; we deliberately chose a “difficult” area in order to
2.95 better contrast pointwise with geometric methods.
2.96 -__________________________
2.97 - 4For each gene, a logistic regression in which the response variable was whether or not a
2.98 -surface pixel was within area AUD, and the predictor variable was the value of the expression
2.99 -of the gene underneath that pixel. The resulting scores were used to rank the genes in terms
2.100 -of how well they predict area AUD.
2.101 - 5For each gene the gradient similarity (see section ??) between (a) a map of the expression
2.102 -of each gene on the cortical surface and (b) the shape of area AUD, was calculated, and this
2.103 -was used to rank the genes.
2.104 - 7
2.105 -
2.106 Principle 4: Work in 2-D whenever possible
2.107 In anatomy, the manifold of interest is usually either defined by a combination
2.108 of two relevant anatomical axes (todo), or by the surface of the structure (as is
2.109 @@ -273,6 +257,23 @@
2.110 the method we develop to include a statistical test that warns the user if the
2.111 assumption of 2-D structure seems to be wrong.
2.112 ——
2.113 +____________________
2.114 + 4For each gene, a logistic regression in which the response variable was whether or not a
2.115 +surface pixel was within area AUD, and the predictor variable was the value of the expression
2.116 +of the gene underneath that pixel. The resulting scores were used to rank the genes in terms
2.117 +of how well they predict area AUD.
2.118 + 5For each gene the gradient similarity (see section ??) between (a) a map of the expression
2.119 +of each gene on the cortical surface and (b) the shape of area AUD, was calculated, and this
2.120 +was used to rank the genes.
2.121 + 7
2.122 +
2.123 +
2.124 +
2.125 + Figure 2: The top row shows the three genes which (individually) best predict
2.126 + area AUD, according to logistic regression. The bottom row shows the three
2.127 + genes which (individually) best match area AUD, according to gradient similar-
2.128 + ity. From left to right and top to bottom, the genes are Ssr1, Efcbp1, Aph1a,
2.129 + Ptk7, Aph1a again, and Lepr
2.130 Massive new datasets obtained with techniques such as in situ hybridization
2.131 (ISH) and BAC-transgenics allow the expression levels of many genes at many
2.132 locations to be compared. This can be used to find marker genes for specific
2.133 @@ -295,10 +296,10 @@
2.134 datasets will be made available in both MATLAB and Caret formats.
2.135 (5) validate the methods developed in (1), (2) and (3) by applying them to
2.136 the cerebral cortex datasets created in (4)
2.137 + 8
2.138 +
2.139 All algorithms that we develop will be implemented in an open-source soft-
2.140 ware toolkit. The toolkit, as well as the machine-readable datasets developed in
2.141 - 8
2.142 -
2.143 aim (4) and any other intermediate dataset we produce, will be published and
2.144 freely available for others to use.
2.145 In addition to developing generally useful methods, the application of these
2.146 @@ -340,11 +341,11 @@
2.147 as a testbed. The Allen Brain Atlas has collected a dataset containing the
2.148 expression level of about 4000 genes* over a set of over 150000 voxels, with a
2.149 spatial resolution of approximately 200 microns[?].
2.150 - We expect to discover sets of marker genes that pick out specific cortical
2.151 -areas. This will allow the development of drugs and other interventions that
2.152 -selectively target individual cortical areas. Therefore our research will lead
2.153 9
2.154
2.155 + We expect to discover sets of marker genes that pick out specific cortical
2.156 + areas. This will allow the development of drugs and other interventions that
2.157 + selectively target individual cortical areas. Therefore our research will lead
2.158 to application in drug discovery, in the development of other targeted clinical
2.159 interventions, and in the development of new experimental techniques.
2.160 The best way to divide up rodent cortex into areas has not been completely
2.161 @@ -388,11 +389,11 @@
2.162 in our publications .
2.163 We also expect to weigh in on the debate about how to best partition rodent
2.164 cortex
2.165 + 10
2.166 +
2.167 be useful for drug discovery as well
2.168 * Another 16000 genes are available, but they do not cover the entire cerebral
2.169 cortex with high spatial resolution.
2.170 - 10
2.171 -
2.172 User-definable ROIs Combinatorial gene expression Negative as well as pos-
2.173 itive signal Use geometry Search for local boundaries if necessary Flatmapped
2.174 Specific aims
2.175 @@ -427,10 +428,10 @@
2.176 matically find the cortical layer boundaries.
2.177 4. Run the procedures that we developed on the cortex: we will present, for
2.178 each area, a short list of markers to identify that area; and we will also
2.179 + 11
2.180 +
2.181 present lists of “panels” of genes that can be used to delineate many areas
2.182 at once.
2.183 - 11
2.184 -
2.185 Develop algorithms to suggest a division of a structure into anatom-
2.186 ical parts
2.187 1. Explore dimensionality reduction algorithms applied to pixels: including
2.188 @@ -471,12 +472,12 @@
2.189 Finder then looks for genes which can distinguish the ROI from the comparator
2.190 region. Specifically, it finds genes for which the ratio (expression energy in the
2.191 ROI) / (expression energy in the comparator region) is high.
2.192 + 12
2.193 +
2.194 Informally, the Gene Finder first infers an ROI based on clustering the seed
2.195 voxel with other voxels. Then, the Gene Finder finds genes which overexpress
2.196 in the ROI as compared to other voxels in the major anatomical region.
2.197 There are three major differences between our approach and Gene Finder.
2.198 - 12
2.199 -
2.200 First, Gene Finder focuses on individual genes and individual ROIs in isola-
2.201 tion. This is great for regions which can be picked out from all other regions by a
2.202 single gene, but not all of them can (todo). There are at least two ways this can
2.203 @@ -519,12 +520,12 @@
2.204 posal. The goal of AGEA’s hierarchial clustering is to generate a binary tree of
2.205 clusters, where a cluster is a collection of voxels. AGEA begins by computing
2.206 the Pearson correlation between each pair of voxels. They then employ a recur-
2.207 + 13
2.208 +
2.209 sive divisive (top-down) hierarchial clustering procedure on the voxels, which
2.210 means that they start with all of the voxels, and then they divide them into clus-
2.211 ters, and then within each cluster, they divide that cluster into smaller clusters,
2.212 etc***. At each step, the collection of voxels is partitioned into two smaller
2.213 - 13
2.214 -
2.215 clusters in a way that maximizes the following quantity: average correlation
2.216 between all possible pairs of voxels containing one voxel from each cluster.
2.217 There are three major differences between our approach and AGEA’s hier-
2.218 @@ -567,12 +568,12 @@
2.219 the performance of our techniques against AGEA’s.
2.220 Another difference between our techniques and AGEA’s is that AGEA allows
2.221 the user to enter only a voxel location, and then to either explore the rest of
2.222 + 14
2.223 +
2.224 the brain’s relationship to that particular voxel, or explore a partitioning of
2.225 the brain based on pairwise voxel correlation. If the user is interested not in a
2.226 single voxel, but rather an entire anatomical structure, AGEA will only succeed
2.227 to the extent that the selected voxel is a typical representative of the structure.
2.228 - 14
2.229 -
2.230 As discussed in the previous paragraph, this poses problems for structures like
2.231 cortical areas, which (because of their division into cortical layers) do not have
2.232 a single “typical representative”.
2.233 @@ -615,12 +616,12 @@
2.234 Despite the distinct roles of different cortical areas in both normal function-
2.235 ing and disease processes, there are no known marker genes for many cortical
2.236 areas. This project will be immediately useful for both drug discovery and clini-
2.237 + 15
2.238 +
2.239 cal research because once the markers are known, interventions can be designed
2.240 which selectively target specific cortical areas.
2.241 This techniques we develop will be useful because they will be applicable to
2.242 the analysis of other anatomical areas, both in terms of finding marker genes
2.243 - 15
2.244 -
2.245 for known areas, and in terms of suggesting new anatomical subdivisions that
2.246 are based upon the gene expression data.
2.247 _______________________________
2.248 @@ -658,12 +659,12 @@
2.249 able to import and export data to standard formats so that users can use our
2.250 software in tandem with other software tools created by other teams. We will
2.251 support the following formats: NIFTI (Neuroimaging Informatics Technology
2.252 -Initiative), SEV (Allen Brain Institute Smoothed Energy Volume), and MAT-
2.253 -LAB. This ensures that our users will not have to exclusively rely on our tools
2.254 -when analyzing data. For example, users will be able to use the data visualiza-
2.255 -tion and analysis capabilities of MATLAB and Caret alongside our software.
2.256 16
2.257
2.258 + Initiative), SEV (Allen Brain Institute Smoothed Energy Volume), and MAT-
2.259 + LAB. This ensures that our users will not have to exclusively rely on our tools
2.260 + when analyzing data. For example, users will be able to use the data visualiza-
2.261 + tion and analysis capabilities of MATLAB and Caret alongside our software.
2.262 To our knowledge, there is no currently available software to convert between
2.263 these formats, so we will also provide a format conversion tool. This may be
2.264 useful even for groups that don’t use any of our other software.
2.265 @@ -705,13 +706,13 @@
2.266 combination of genes are expressed, the local tissue is probably part of a certain
2.267 subregion. This means that we can then confidentally develop an intervention
2.268 which is triggered only when that combination of genes are expressed; and to
2.269 + 17
2.270 +
2.271 the extent that the result procedure is reliable, we know that the intervention
2.272 will only be triggered in the target subregion.
2.273 We said that the result procedure provides “a way to use the gene expression
2.274 profiles of voxels in a tissue sample” in order to “determine where the subregions
2.275 are”.
2.276 - 17
2.277 -
2.278 Does the result procedure get as input all of the gene expression profiles
2.279 of each voxel in the entire tissue sample, and produce as output all of the
2.280 subregional boundaries all at once?
2.281 @@ -751,12 +752,12 @@
2.282 if multiple subregions are present, where they each are. Or it can be used
2.283 indirectly; imagine that the result procedure tells us that whenever a certain
2.284 combination of genes are expressed, the local tissue is probably part of a certain
2.285 + 18
2.286 +
2.287 subregion. This means that we can then confidentally develop an intervention
2.288 which is triggered only when that combination of genes are expressed; and to
2.289 the extent that the result procedure is reliable, we know that the intervention
2.290 will only be triggered in the target subregion.
2.291 - 18
2.292 -
2.293 We said that the result procedure provides “a way to use the gene expression
2.294 profiles of voxels in a tissue sample” in order to “determine where the subregions
2.295 are”.
3.1 Binary file grant.odt has changed
4.1 --- a/grant.txt Sat Apr 11 19:59:01 2009 -0700
4.2 +++ b/grant.txt Sat Apr 11 21:31:27 2009 -0700
4.3 @@ -1,5 +1,7 @@
4.4 == Specific aims ==
4.5
4.6 +todo test3
4.7 +
4.8 Massive new datasets obtained with techniques such as in situ hybridization (ISH) and BAC-transgenics allow the expression levels of many genes at many locations to be compared. Our goal is to develop automated methods to relate spatial variation in gene expression to anatomy. We want to find marker genes for specific anatomical regions, and also to draw new anatomical maps based on gene expression patterns. We have three specific aims:
4.9
4.10 (1) develop an algorithm to screen spatial gene expression data for combinations of marker genes which selectively target anatomical regions