From ee82b1d72735d7689e741718d3a169568cab7441 Mon Sep 17 00:00:00 2001 From: Colin Date: Sun, 5 Mar 2023 04:13:18 -0700 Subject: [PATCH] Add CoreGetRegions --- packages/core/assemblyManager/assembly.ts | 233 ++++++++---------- .../assemblyManager/assemblyConfigSchema.ts | 29 ++- .../core/assemblyManager/assemblyManager.ts | 27 +- .../core/assemblyManager/loadRefNameMap.ts | 65 +++++ packages/core/assemblyManager/util.ts | 69 ++++++ .../models/BaseTrackModel.ts | 1 - .../models/components/SaveTrackData.tsx | 59 ++--- packages/core/rpc/coreRpcMethods.ts | 3 +- packages/core/rpc/methods/CoreGetRefNames.ts | 8 +- packages/core/rpc/methods/CoreGetRegions.ts | 26 ++ .../src/LinearGenomeView/model.ts | 4 + test_data/volvox/config.json | 3 - 12 files changed, 327 insertions(+), 200 deletions(-) create mode 100644 packages/core/assemblyManager/loadRefNameMap.ts create mode 100644 packages/core/assemblyManager/util.ts create mode 100644 packages/core/rpc/methods/CoreGetRegions.ts diff --git a/packages/core/assemblyManager/assembly.ts b/packages/core/assemblyManager/assembly.ts index dd85201d89..23beb23a85 100644 --- a/packages/core/assemblyManager/assembly.ts +++ b/packages/core/assemblyManager/assembly.ts @@ -1,20 +1,23 @@ import { getParent, types, Instance, IAnyType } from 'mobx-state-tree' -import jsonStableStringify from 'json-stable-stringify' import AbortablePromiseCache from 'abortable-promise-cache' // locals -import { getConf, AnyConfigurationModel } from '../configuration' -import { - BaseRefNameAliasAdapter, - RegionsAdapter, -} from '../data_adapters/BaseAdapter' +import { getConf } from '../configuration' + import PluginManager from '../PluginManager' -import { when, Region, Feature } from '../util' +import { Region, Feature } from '../util' import QuickLRU from '../util/QuickLRU' - -const refNameRegex = new RegExp( - '[0-9A-Za-z!#$%&+./:;?@^_|~-][0-9A-Za-z!#$%&*+./:;=?@^_|~-]*', -) +import { + BaseOptions, + checkRefName, + getAdapterId, + getAssemblyRegions, + getCytobands, + getRefNameAliases, + RefNameAliases, +} from './util' +import { BasicRegion, loadRefNameMap } from './loadRefNameMap' +import RpcManager from '../rpc/RpcManager' // Based on the UCSC Genome Browser chromosome color palette: // https://github.com/ucscGenomeBrowser/kent/blob/a50ed53aff81d6fb3e34e6913ce18578292bc24e/src/hg/inc/chromColors.h @@ -49,74 +52,6 @@ const refNameColors = [ 'rgb(96, 163, 48)', // originally 'rgb(121, 204, 61)' ] -async function loadRefNameMap( - assembly: Assembly, - adapterConfig: unknown, - options: BaseOptions, - signal?: AbortSignal, -) { - const { sessionId } = options - await when(() => !!(assembly.regions && assembly.refNameAliases), { - signal, - name: 'when assembly ready', - }) - - const refNames = (await assembly.rpcManager.call( - sessionId, - 'CoreGetRefNames', - { - adapterConfig, - signal, - ...options, - }, - { timeout: 1000000 }, - )) as string[] - - const { refNameAliases } = assembly - if (!refNameAliases) { - throw new Error(`error loading assembly ${assembly.name}'s refNameAliases`) - } - - const refNameMap = Object.fromEntries( - refNames.map(name => { - checkRefName(name) - return [assembly.getCanonicalRefName(name), name] - }), - ) - - // make the reversed map too - const reversed = Object.fromEntries( - Object.entries(refNameMap).map(([canonicalName, adapterName]) => [ - adapterName, - canonicalName, - ]), - ) - - return { - forwardMap: refNameMap, - reverseMap: reversed, - } -} - -// Valid refName pattern from https://samtools.github.io/hts-specs/SAMv1.pdf -function checkRefName(refName: string) { - if (!refNameRegex.test(refName)) { - throw new Error(`Encountered invalid refName: "${refName}"`) - } -} - -function getAdapterId(adapterConf: unknown) { - return jsonStableStringify(adapterConf) -} - -type RefNameAliases = Record - -export interface BaseOptions { - signal?: AbortSignal - sessionId: string - statusCallback?: Function -} - interface CacheData { adapterConf: unknown self: Assembly @@ -129,19 +64,16 @@ export interface RefNameMap { reverseMap: RefNameAliases } -export interface BasicRegion { - start: number - end: number - refName: string - assemblyName: string -} - export interface Loading { adapterRegionsWithAssembly: Region[] refNameAliases: RefNameAliases lowerCaseRefNameAliases: RefNameAliases cytobands: Feature[] } + +/** + * #stateModel Assembly + */ export default function assemblyFactory( assemblyConfigType: IAnyType, pm: PluginManager, @@ -164,6 +96,9 @@ export default function assemblyFactory( }) return types .model({ + /** + * #slot + */ configuration: types.safeReference(assemblyConfigType), }) .volatile(() => ({ @@ -176,72 +111,107 @@ export default function assemblyFactory( cytobands: undefined as Feature[] | undefined, })) .views(self => ({ + /** + * #getter + */ get initialized() { // @ts-expect-error self.load() return !!self.refNameAliases }, + /** + * #getter + */ get name(): string { return getConf(self, 'name') }, - + /** + * #getter + */ get regions() { // @ts-expect-error self.load() return self.volatileRegions }, - + /** + * #getter + */ get aliases(): string[] { return getConf(self, 'aliases') }, - + /** + * #getter + */ get displayName(): string | undefined { return getConf(self, 'displayName') }, - + /** + * #getter + */ hasName(name: string) { return this.allAliases.includes(name) }, - + /** + * #getter + */ get allAliases() { return [this.name, ...this.aliases] }, - - // note: lowerCaseRefNameAliases not included here: this allows the list - // of refnames to be just the "normal casing", but things like - // getCanonicalRefName can resolve a lower-case name if needed + /** + * #getter + * note: lowerCaseRefNameAliases not included here: this allows the list + * of refnames to be just the "normal casing", but things like + * getCanonicalRefName can resolve a lower-case name if needed + */ get allRefNames() { return !self.refNameAliases ? undefined : Object.keys(self.refNameAliases) }, - + /** + * #getter + */ get lowerCaseRefNames() { return !self.lowerCaseRefNameAliases ? undefined : Object.keys(self.lowerCaseRefNameAliases || {}) }, + /** + * #getter + */ get allRefNamesWithLowerCase() { return this.allRefNames && this.lowerCaseRefNames ? [...new Set([...this.allRefNames, ...this.lowerCaseRefNames])] : undefined }, - get rpcManager() { + /** + * #getter + */ + get rpcManager(): RpcManager { // eslint-disable-next-line @typescript-eslint/no-explicit-any return getParent(self, 2).rpcManager }, + /** + * #getter + */ get refNameColors() { const colors: string[] = getConf(self, 'refNameColors') return colors.length === 0 ? refNameColors : colors }, })) .views(self => ({ + /** + * #getter + */ get refNames() { return self.regions?.map(region => region.refName) }, })) .views(self => ({ + /** + * #method + */ getCanonicalRefName(refName: string) { if (!self.refNameAliases || !self.lowerCaseRefNameAliases) { throw new Error( @@ -252,6 +222,9 @@ export default function assemblyFactory( self.refNameAliases[refName] || self.lowerCaseRefNameAliases[refName] ) }, + /** + * #method + */ getRefNameColor(refName: string) { if (!self.refNames) { return undefined @@ -262,6 +235,9 @@ export default function assemblyFactory( } return self.refNameColors[idx % self.refNameColors.length] }, + /** + * #method + */ isValidRefName(refName: string) { if (!self.refNameAliases) { throw new Error( @@ -272,6 +248,9 @@ export default function assemblyFactory( }, })) .actions(self => ({ + /** + * #action + */ setLoaded({ adapterRegionsWithAssembly, refNameAliases, @@ -283,23 +262,41 @@ export default function assemblyFactory( this.setRefNameAliases(refNameAliases, lowerCaseRefNameAliases) this.setCytobands(cytobands) }, + /** + * #action + */ setError(e: unknown) { console.error(e) self.error = e }, + /** + * #action + */ setRegions(regions: Region[]) { self.volatileRegions = regions }, + /** + * #action + */ setRefNameAliases(aliases: RefNameAliases, lcAliases: RefNameAliases) { self.refNameAliases = aliases self.lowerCaseRefNameAliases = lcAliases }, + /** + * #action + */ setCytobands(cytobands: Feature[]) { self.cytobands = cytobands }, + /** + * #action + */ setLoadingP(p?: Promise) { self.loadingP = p }, + /** + * #action + */ load() { if (!self.loadingP) { self.loadingP = this.loadPre().catch(e => { @@ -309,6 +306,9 @@ export default function assemblyFactory( } return self.loadingP }, + /** + * #action + */ async loadPre() { const conf = self.configuration const refNameAliasesAdapterConf = conf.refNameAliases?.adapter @@ -316,7 +316,8 @@ export default function assemblyFactory( const sequenceAdapterConf = conf.sequence.adapter const assemblyName = self.name - const regions = await getAssemblyRegions(sequenceAdapterConf, pm) + const regions = await getAssemblyRegions(self, sequenceAdapterConf) + console.log({ regions }) const adapterRegionsWithAssembly = regions.map(r => { checkRefName(r.refName) return { ...r, assemblyName } @@ -352,6 +353,9 @@ export default function assemblyFactory( }, })) .views(self => ({ + /** + * #method + */ getAdapterMapEntry(adapterConf: unknown, options: BaseOptions) { const { signal, statusCallback, ...rest } = options if (!options.sessionId) { @@ -374,6 +378,7 @@ export default function assemblyFactory( }, /** + * #method * get Map of `canonical-name -> adapter-specific-name` */ async getRefNameMapForAdapter(adapterConf: unknown, opts: BaseOptions) { @@ -385,6 +390,7 @@ export default function assemblyFactory( }, /** + * #method * get Map of `adapter-specific-name -> canonical-name` */ async getReverseRefNameMapForAdapter( @@ -397,36 +403,5 @@ export default function assemblyFactory( })) } -async function getRefNameAliases( - config: AnyConfigurationModel, - pm: PluginManager, - signal?: AbortSignal, -) { - const type = pm.getAdapterType(config.type) - const CLASS = await type.getAdapterClass() - const adapter = new CLASS(config, undefined, pm) as BaseRefNameAliasAdapter - return adapter.getRefNameAliases({ signal }) -} - -async function getCytobands(config: AnyConfigurationModel, pm: PluginManager) { - const type = pm.getAdapterType(config.type) - const CLASS = await type.getAdapterClass() - const adapter = new CLASS(config, undefined, pm) - - // @ts-expect-error - return adapter.getData() -} - -async function getAssemblyRegions( - config: AnyConfigurationModel, - pm: PluginManager, - signal?: AbortSignal, -) { - const type = pm.getAdapterType(config.type) - const CLASS = await type.getAdapterClass() - const adapter = new CLASS(config, undefined, pm) as RegionsAdapter - return adapter.getRegions({ signal }) -} - export type AssemblyModel = ReturnType export type Assembly = Instance diff --git a/packages/core/assemblyManager/assemblyConfigSchema.ts b/packages/core/assemblyManager/assemblyConfigSchema.ts index 2e93958d0a..398fa46d47 100644 --- a/packages/core/assemblyManager/assemblyConfigSchema.ts +++ b/packages/core/assemblyManager/assemblyConfigSchema.ts @@ -2,10 +2,11 @@ import { ConfigurationSchema } from '../configuration' import PluginManager from '../PluginManager' /** - * #config BaseAssembly - * This corresponds to the assemblies section of the config + * #config Assembly + * This corresponds to the assemblies section of the config, generally accessed + * through the assemblyManager */ -function assemblyConfigSchema(pluginManager: PluginManager) { +export default function assemblyConfigSchema(pluginManager: PluginManager) { return ConfigurationSchema( 'BaseAssembly', { @@ -21,8 +22,8 @@ function assemblyConfigSchema(pluginManager: PluginManager) { /** * #slot - * sequence refers to a reference sequence track that has an adapter containing, - * importantly, a sequence adapter such as IndexedFastaAdapter + * sequence refers to a reference sequence track that has an adapter + * containing, importantly, a sequence adapter such as IndexedFastaAdapter */ sequence: pluginManager.getTrackType('ReferenceSequenceTrack') .configSchema, @@ -42,9 +43,9 @@ function assemblyConfigSchema(pluginManager: PluginManager) { { /** * #slot refNameAliases.adapter - * refNameAliases help resolve e.g. chr1 and 1 as the same entity - * the data for refNameAliases are fetched from an adapter, that is - * commonly a tsv like chromAliases.txt from UCSC or similar + * refNameAliases help resolve e.g. chr1 and 1 as the same entity the + * data for refNameAliases are fetched from an adapter, that is commonly a tsv + * like chromAliases.txt from UCSC or similar */ adapter: pluginManager.pluggableConfigSchemaType('adapter'), }, @@ -64,8 +65,8 @@ function assemblyConfigSchema(pluginManager: PluginManager) { { /** * #slot cytobands.adapter - * cytoband data is fetched from an adapter, and can be displayed by a - * view type as ideograms + * cytoband data is fetched from an adapter, and can be displayed by + * a view type as ideograms */ adapter: pluginManager.pluggableConfigSchemaType('adapter'), }, @@ -94,13 +95,11 @@ function assemblyConfigSchema(pluginManager: PluginManager) { { /** * #identifier name - * the name acts as a unique identifier in the config, so it cannot be duplicated. - * it usually a short human readable "id" like hg38, but you can also optionally - * customize the assembly "displayName" config slot + * the name acts as a unique identifier in the config, so it cannot be + * duplicated. it usually a short human readable "id" like hg38, but you can + * also optionally customize the assembly "displayName" config slot */ explicitIdentifier: 'name', }, ) } - -export default assemblyConfigSchema diff --git a/packages/core/assemblyManager/assemblyManager.ts b/packages/core/assemblyManager/assemblyManager.ts index 8544139557..cf4e70e0c9 100644 --- a/packages/core/assemblyManager/assemblyManager.ts +++ b/packages/core/assemblyManager/assemblyManager.ts @@ -6,13 +6,17 @@ import { Instance, IAnyType, } from 'mobx-state-tree' -import { when } from '../util' import { reaction } from 'mobx' +// locals +import { when } from '../util' import { readConfObject, AnyConfigurationModel } from '../configuration' import assemblyFactory, { Assembly } from './assembly' import PluginManager from '../PluginManager' -function assemblyManagerFactory(conf: IAnyType, pm: PluginManager) { +export default function assemblyManagerFactory( + conf: IAnyType, + pm: PluginManager, +) { type Conf = Instance | string return types .model({ @@ -116,21 +120,20 @@ function assemblyManagerFactory(conf: IAnyType, pm: PluginManager) { addDisposer( self, reaction( - // have to slice it to be properly reacted to () => self.assemblyList, - assemblyConfigs => { + confs => { self.assemblies.forEach(asm => { if (!asm.configuration) { this.removeAssembly(asm) } }) - assemblyConfigs.forEach(assemblyConfig => { - const existingAssemblyIdx = self.assemblies.findIndex( - assembly => - assembly.name === readConfObject(assemblyConfig, 'name'), - ) - if (existingAssemblyIdx === -1) { - this.addAssembly(assemblyConfig) + confs.forEach(conf => { + if ( + !self.assemblies.some( + a => a.name === readConfObject(conf, 'name'), + ) + ) { + this.addAssembly(conf) } }) }, @@ -154,5 +157,3 @@ function assemblyManagerFactory(conf: IAnyType, pm: PluginManager) { }, })) } - -export default assemblyManagerFactory diff --git a/packages/core/assemblyManager/loadRefNameMap.ts b/packages/core/assemblyManager/loadRefNameMap.ts new file mode 100644 index 0000000000..67928d2b7c --- /dev/null +++ b/packages/core/assemblyManager/loadRefNameMap.ts @@ -0,0 +1,65 @@ +import { BaseOptions, checkRefName, RefNameAliases } from './util' +import RpcManager from '../rpc/RpcManager' +import { when } from '../util' + +export interface BasicRegion { + start: number + end: number + refName: string + assemblyName: string +} + +export async function loadRefNameMap( + assembly: { + name: string + regions: BasicRegion[] | undefined + refNameAliases: RefNameAliases | undefined + getCanonicalRefName: (arg: string) => string + rpcManager: RpcManager + }, + adapterConfig: unknown, + options: BaseOptions, + signal?: AbortSignal, +) { + const { sessionId } = options + await when(() => !!(assembly.regions && assembly.refNameAliases), { + signal, + name: 'when assembly ready', + }) + + const refNames = (await assembly.rpcManager.call( + sessionId, + 'CoreGetRefNames', + { + adapterConfig, + signal, + ...options, + }, + { timeout: 1000000 }, + )) as string[] + + const { refNameAliases } = assembly + if (!refNameAliases) { + throw new Error(`error loading assembly ${assembly.name}'s refNameAliases`) + } + + const refNameMap = Object.fromEntries( + refNames.map(name => { + checkRefName(name) + return [assembly.getCanonicalRefName(name), name] + }), + ) + + // make the reversed map too + const reversed = Object.fromEntries( + Object.entries(refNameMap).map(([canonicalName, adapterName]) => [ + adapterName, + canonicalName, + ]), + ) + + return { + forwardMap: refNameMap, + reverseMap: reversed, + } +} diff --git a/packages/core/assemblyManager/util.ts b/packages/core/assemblyManager/util.ts new file mode 100644 index 0000000000..7b75ec5273 --- /dev/null +++ b/packages/core/assemblyManager/util.ts @@ -0,0 +1,69 @@ +import { AnyConfigurationModel } from '../configuration' +import jsonStableStringify from 'json-stable-stringify' +import { BaseRefNameAliasAdapter } from '../data_adapters/BaseAdapter' +import PluginManager from '../PluginManager' +import { BasicRegion } from './loadRefNameMap' + +export type RefNameAliases = Record + +export interface BaseOptions { + signal?: AbortSignal + sessionId: string + statusCallback?: Function +} + +export async function getRefNameAliases( + config: AnyConfigurationModel, + pm: PluginManager, + signal?: AbortSignal, +) { + const type = pm.getAdapterType(config.type) + const CLASS = await type.getAdapterClass() + const adapter = new CLASS(config, undefined, pm) as BaseRefNameAliasAdapter + return adapter.getRefNameAliases({ signal }) +} + +export async function getCytobands( + config: AnyConfigurationModel, + pm: PluginManager, +) { + const type = pm.getAdapterType(config.type) + const CLASS = await type.getAdapterClass() + const adapter = new CLASS(config, undefined, pm) + + // @ts-expect-error + return adapter.getData() +} + +export async function getAssemblyRegions( + assembly: any, + adapterConfig: AnyConfigurationModel, + signal?: AbortSignal, +): Promise { + const sessionId = 'loadRefNames' + return assembly.rpcManager.call( + sessionId, + 'CoreGetRegions', + { + adapterConfig, + sessionId, + signal, + }, + { timeout: 1000000 }, + ) +} + +const refNameRegex = new RegExp( + '[0-9A-Za-z!#$%&+./:;?@^_|~-][0-9A-Za-z!#$%&*+./:;=?@^_|~-]*', +) + +// Valid refName pattern from https://samtools.github.io/hts-specs/SAMv1.pdf +export function checkRefName(refName: string) { + if (!refNameRegex.test(refName)) { + throw new Error(`Encountered invalid refName: "${refName}"`) + } +} + +export function getAdapterId(adapterConf: unknown) { + return jsonStableStringify(adapterConf) +} diff --git a/packages/core/pluggableElementTypes/models/BaseTrackModel.ts b/packages/core/pluggableElementTypes/models/BaseTrackModel.ts index 17964e9017..5885a29019 100644 --- a/packages/core/pluggableElementTypes/models/BaseTrackModel.ts +++ b/packages/core/pluggableElementTypes/models/BaseTrackModel.ts @@ -221,7 +221,6 @@ export function createBaseTrackModel( onClick: () => { getSession(self).queueDialog(handleClose => [ SaveTrackDataDlg, - // @ts-ignore { model: self, handleClose }, ]) }, diff --git a/packages/core/pluggableElementTypes/models/components/SaveTrackData.tsx b/packages/core/pluggableElementTypes/models/components/SaveTrackData.tsx index cb1390a5f8..e32f1ec083 100644 --- a/packages/core/pluggableElementTypes/models/components/SaveTrackData.tsx +++ b/packages/core/pluggableElementTypes/models/components/SaveTrackData.tsx @@ -11,6 +11,7 @@ import { TextField, Typography, } from '@mui/material' +import { IAnyStateTreeNode } from 'mobx-state-tree' import { makeStyles } from 'tss-react/mui' import { saveAs } from 'file-saver' import { observer } from 'mobx-react' @@ -22,7 +23,6 @@ import { Region, } from '@jbrowse/core/util' import { getConf } from '@jbrowse/core/configuration' -import { BaseTrackModel } from '@jbrowse/core/pluggableElementTypes' // icons import GetAppIcon from '@mui/icons-material/GetApp' @@ -41,7 +41,7 @@ const useStyles = makeStyles()({ }) async function fetchFeatures( - track: BaseTrackModel, + track: IAnyStateTreeNode, regions: Region[], signal?: AbortSignal, ) { @@ -60,7 +60,7 @@ export default observer(function SaveTrackDataDlg({ model, handleClose, }: { - model: BaseTrackModel + model: IAnyStateTreeNode handleClose: () => void }) { const { classes } = useStyles() @@ -77,11 +77,9 @@ export default observer(function SaveTrackDataDlg({ // eslint-disable-next-line @typescript-eslint/no-floating-promises ;(async () => { try { - const view = getContainingView(model) + const view = getContainingView(model) as { visibleRegions?: Region[] } setError(undefined) - setFeatures( - await fetchFeatures(model, view.dynamicBlocks.contentBlocks), - ) + setFeatures(await fetchFeatures(model, view.visibleRegions || [])) } catch (e) { console.error(e) setError(e) @@ -92,20 +90,24 @@ export default observer(function SaveTrackDataDlg({ useEffect(() => { // eslint-disable-next-line @typescript-eslint/no-floating-promises ;(async () => { - const view = getContainingView(model) - const session = getSession(model) - if (!features) { - return - } - const str = await (type === 'gff3' - ? stringifyGFF3(features) - : stringifyGenbank({ - features, - session, - assemblyName: view.dynamicBlocks.contentBlocks[0].assemblyName, - })) + try { + const view = getContainingView(model) + const session = getSession(model) + if (!features) { + return + } + const str = await (type === 'gff3' + ? stringifyGFF3(features) + : stringifyGenbank({ + features, + session, + assemblyName: view.dynamicBlocks.contentBlocks[0].assemblyName, + })) - setStr(str) + setStr(str) + } catch (e) { + setError(e) + } })() }, [type, features, model]) @@ -121,10 +123,7 @@ export default observer(function SaveTrackDataDlg({ File type - setType(event.target.value)} - > + setType(e.target.value)}> {Object.entries(options).map(([key, val]) => (