feat: initial DWARF debugger integration

Enables stepping around and setting breakpoints in native code.
Variables represenation is still a todo, and this needs some polish.

It refactors the `Source.sourceMap` into a more generic location
provider, which may be backed by the new IWasmSymbols interface. This
then talks to the DWARF debugger. The SourceContainer's `_sourceMaps`
are also now just folded into the `Source.sourceMap`; that information
was duplicate and the only benefit was deduplication if two sources had
the same sourcemap, but this is really rare.

This also made all location mapping asynchronous, so there were a
few refactors around there, too.

It also refactors how I implemented WAT (text format decompilation) last
iteration. That previously "pretended" the source of the WASM was WAT,
but this caused issues because the location transformation we did on the
scripts is done before location mapping. So instead, the WAT is an extra
fake 'file' in the symbols and we resolve any unknown locations into the
WAT file.

One that that _doesn't_ work yet is entrypoint breakpoints for wasm,
which means that breakpoints set before the debug session starts may be
missed. I have a thread out to the Chromium folks to see if there's
a solution to this.

For #1789

.nvmrc

@@ -0,0 +1 @@
+18

.vscode/settings.json

@@ -28,5 +28,5 @@
     "source.organizeImports": true
   },
   "editor.formatOnSave": true,
-  "python.formatting.provider": "black"
+  "typescript.tsserver.experimental.enableProjectDiagnostics": true,
 }

gulpfile.js

@@ -24,6 +24,7 @@ const pipelineAsync = util.promisify(stream.pipeline);
 
 const dirname = 'js-debug';
 const sources = ['src/**/*.{ts,tsx}'];
+const externalModules = ['@vscode/dwarf-debugging'];
 const allPackages = [];
 
 const srcDir = 'src';
@@ -232,7 +233,7 @@ async function compileTs({
         resolveExtensions: isInVsCode
           ? ['.extensionOnly.ts', ...resolveDefaultExts]
           : resolveDefaultExts,
-        external: isInVsCode ? ['vscode'] : [],
+        external: isInVsCode ? ['vscode', ...externalModules] : externalModules,
         sourcemap: !!sourcemap,
         sourcesContent: false,
         packages: nodePackages,

src/adapter/breakpoints.ts

@@ -5,7 +5,7 @@
 import { inject, injectable } from 'inversify';
 import Cdp from '../cdp/api';
 import { ILogger, LogTag } from '../common/logging';
-import { bisectArray, flatten } from '../common/objUtils';
+import { bisectArrayAsync, flatten } from '../common/objUtils';
 import { IPosition } from '../common/positions';
 import { delay } from '../common/promiseUtil';
 import { SourceMap } from '../common/sourceMaps/sourceMap';
@@ -25,15 +25,8 @@ import { NeverResolvedBreakpoint } from './breakpoints/neverResolvedBreakpoint';
 import { PatternEntryBreakpoint } from './breakpoints/patternEntrypointBreakpoint';
 import { UserDefinedBreakpoint } from './breakpoints/userDefinedBreakpoint';
 import { DiagnosticToolSuggester } from './diagnosticToolSuggester';
-import {
-  base0To1,
-  base1To0,
-  ISourceWithMap,
-  isSourceWithMap,
-  IUiLocation,
-  Source,
-  SourceContainer,
-} from './sources';
+import { ISourceWithMap, IUiLocation, Source, base0To1, base1To0, isSourceWithMap } from './source';
+import { SourceContainer } from './sourceContainer';
 import { ScriptWithSourceMapHandler, Thread } from './threads';
 
 /**
@@ -226,23 +219,30 @@ export class BreakpointManager {
    * location in the `toSource`, using the provided source map. Breakpoints
    * are don't have a corresponding location won't be moved.
    */
-  public moveBreakpoints(fromSource: Source, sourceMap: SourceMap, toSource: Source) {
+  public async moveBreakpoints(
+    thread: Thread,
+    fromSource: Source,
+    sourceMap: SourceMap,
+    toSource: Source,
+  ) {
     const tryUpdateLocations = (breakpoints: UserDefinedBreakpoint[]) =>
-      bisectArray(breakpoints, bp => {
-        const gen = this._sourceContainer.getOptiminalOriginalPosition(
+      bisectArrayAsync(breakpoints, async bp => {
+        const gen = await this._sourceContainer.getOptiminalOriginalPosition(
           sourceMap,
           bp.originalPosition,
         );
-        if (gen.column === null || gen.line === null) {
+        if (!gen) {
           return false;
         }
 
+        const base1 = gen.position.base1;
         bp.updateSourceLocation(
+          thread,
           {
             path: toSource.absolutePath,
             sourceReference: toSource.sourceReference,
           },
-          { lineNumber: gen.line, columnNumber: gen.column + 1, source: toSource },
+          { lineNumber: base1.lineNumber, columnNumber: base1.columnNumber, source: toSource },
         );
         return false;
       });
@@ -251,14 +251,14 @@ export class BreakpointManager {
     const toPath = toSource.absolutePath;
     const byPath = fromPath ? this._byPath.get(fromPath) : undefined;
     if (byPath && toPath) {
-      const [remaining, moved] = tryUpdateLocations(byPath);
+      const [remaining, moved] = await tryUpdateLocations(byPath);
       this._byPath.set(fromPath, remaining);
       this._byPath.set(toPath, moved);
     }
 
     const byRef = this._byRef.get(fromSource.sourceReference);
     if (byRef) {
-      const [remaining, moved] = tryUpdateLocations(byRef);
+      const [remaining, moved] = await tryUpdateLocations(byRef);
       this._byRef.set(fromSource.sourceReference, remaining);
       this._byRef.set(toSource.sourceReference, moved);
     }
@@ -317,18 +317,19 @@ export class BreakpointManager {
     end: IPosition,
   ) {
     const start1 = start.base1;
-    const startLocations = this._sourceContainer.currentSiblingUiLocations({
-      source,
-      lineNumber: start1.lineNumber,
-      columnNumber: start1.columnNumber,
-    });
-
     const end1 = end.base1;
-    const endLocations = this._sourceContainer.currentSiblingUiLocations({
-      source,
-      lineNumber: end1.lineNumber,
-      columnNumber: end1.columnNumber,
-    });
+    const [startLocations, endLocations] = await Promise.all([
+      this._sourceContainer.currentSiblingUiLocations({
+        source,
+        lineNumber: start1.lineNumber,
+        columnNumber: start1.columnNumber,
+      }),
+      this._sourceContainer.currentSiblingUiLocations({
+        source,
+        lineNumber: end1.lineNumber,
+        columnNumber: end1.columnNumber,
+      }),
+    ]);
 
     // As far as I know the number of start and end locations should be the
     // same, log if this is not the case.
@@ -343,7 +344,7 @@ export class BreakpointManager {
     // For each viable location, attempt to identify its script ID and then ask
     // Chrome for the breakpoints in the given range. For almost all scripts
     // we'll only every find one viable location with a script.
-    const todo: Promise<void>[] = [];
+    const todo: Promise<unknown>[] = [];
     const result: IPossibleBreakLocation[] = [];
     for (let i = 0; i < startLocations.length; i++) {
       const start = startLocations[i];
@@ -383,15 +384,17 @@ export class BreakpointManager {
             // Discard any that map outside of the source we're interested in,
             // which is possible (e.g. if a section of code from one source is
             // inlined amongst the range we request).
-            for (const breakLocation of r.locations) {
-              const { lineNumber, columnNumber = 0 } = breakLocation;
-              const uiLocations = this._sourceContainer.currentSiblingUiLocations({
-                source: lsrc,
-                ...lsrc.offsetScriptToSource(base0To1({ lineNumber, columnNumber })),
-              });
-
-              result.push({ breakLocation, uiLocations });
-            }
+            return Promise.all(
+              r.locations.map(async breakLocation => {
+                const { lineNumber, columnNumber = 0 } = breakLocation;
+                const uiLocations = await this._sourceContainer.currentSiblingUiLocations({
+                  source: lsrc,
+                  ...lsrc.offsetScriptToSource(base0To1({ lineNumber, columnNumber })),
+                });
+
+                result.push({ breakLocation, uiLocations });
+              }),
+            );
           }),
       );
     }
@@ -671,7 +674,7 @@ export class BreakpointManager {
   }
 
   /**
-   * Rreturns whether any of the given breakpoints are an entrypoint breakpoint.
+   * Returns whether any of the given breakpoints are an entrypoint breakpoint.
    */
   public isEntrypointBreak(
     hitBreakpointIds: ReadonlyArray<Cdp.Debugger.BreakpointId>,

src/adapter/breakpoints/breakpointBase.ts

@@ -4,17 +4,11 @@
 
 import Cdp from '../../cdp/api';
 import { LogTag } from '../../common/logging';
+import { IPosition } from '../../common/positions';
 import { absolutePathToFileUrl, urlToRegex } from '../../common/urlUtils';
 import Dap from '../../dap/api';
 import { BreakpointManager } from '../breakpoints';
-import {
-  base1To0,
-  ISourceScript,
-  IUiLocation,
-  Source,
-  SourceFromMap,
-  WasmSource,
-} from '../sources';
+import { ISourceScript, IUiLocation, Source, SourceFromMap, base1To0 } from '../source';
 import { Script, Thread } from '../threads';
 
 export type LineColumn = { lineNumber: number; columnNumber: number }; // 1-based
@@ -147,20 +141,18 @@ export abstract class Breakpoint {
    * the breakpoints when we pretty print a source. This is dangerous with
    * sharp edges, use with caution.
    */
-  public async updateSourceLocation(source: Dap.Source, uiLocation: IUiLocation) {
+  public async updateSourceLocation(thread: Thread, source: Dap.Source, uiLocation: IUiLocation) {
     this._source = source;
     this._originalPosition = uiLocation;
 
-    this.updateCdpRefs(list =>
-      list.map(bp =>
-        bp.state === CdpReferenceState.Applied
-          ? {
-              ...bp,
-              uiLocations: this._manager._sourceContainer.currentSiblingUiLocations(uiLocation),
-            }
-          : bp,
-      ),
-    );
+    const todo: Promise<unknown>[] = [];
+    for (const ref of this.cdpBreakpoints) {
+      if (ref.state === CdpReferenceState.Applied) {
+        todo.push(this.updateUiLocations(thread, ref.cdpId, ref.locations));
+      }
+    }
+
+    await Promise.all(todo);
   }
 
   /**
@@ -205,7 +197,7 @@ export abstract class Breakpoint {
 
     // double check still enabled to avoid racing
     if (source && this.isEnabled) {
-      const uiLocations = this._manager._sourceContainer.currentSiblingUiLocations({
+      const uiLocations = await this._manager._sourceContainer.currentSiblingUiLocations({
         lineNumber: this.originalPosition.lineNumber,
         columnNumber: this.originalPosition.columnNumber,
         source,
@@ -250,8 +242,9 @@ export abstract class Breakpoint {
       return;
     }
 
+    const locations = await this._manager._sourceContainer.currentSiblingUiLocations(uiLocation);
+
     this.updateExistingCdpRef(cdpId, bp => {
-      const locations = this._manager._sourceContainer.currentSiblingUiLocations(uiLocation);
       const inPreferredSource = locations.filter(l => l.source === source);
       return {
         ...bp,
@@ -305,12 +298,8 @@ export abstract class Breakpoint {
       return [];
     }
 
-    if (source instanceof WasmSource) {
-      await source.offsetsAssembled;
-    }
-
     // Find all locations for this breakpoint in the new script.
-    const uiLocations = this._manager._sourceContainer.currentSiblingUiLocations(
+    const uiLocations = await this._manager._sourceContainer.currentSiblingUiLocations(
       {
         lineNumber: this.originalPosition.lineNumber,
         columnNumber: this.originalPosition.columnNumber,
@@ -374,6 +363,24 @@ export abstract class Breakpoint {
     this.updateCdpRefs(l => l.filter(bp => isSetByUrl(bp.args)));
   }
 
+  /**
+   * Gets whether this breakpoint has resolved to the given position.
+   */
+  public hasResolvedAt(scriptId: string, position: IPosition) {
+    const { lineNumber, columnNumber } = position.base0;
+
+    return this.cdpBreakpoints.some(
+      bp =>
+        bp.state === CdpReferenceState.Applied &&
+        bp.locations.some(
+          l =>
+            l.scriptId === scriptId &&
+            l.lineNumber === lineNumber &&
+            (l.columnNumber === undefined || l.columnNumber === columnNumber),
+        ),
+    );
+  }
+
   /**
    * Gets whether the breakpoint was set in the source by URL. Also checks
    * the rebased remote paths, since Sources are always normalized to the
@@ -566,7 +573,10 @@ export abstract class Breakpoint {
     urlRegex: string,
     lineColumn: LineColumn,
   ): Promise<void> {
-    lineColumn = base1To0(lineColumn);
+    lineColumn = {
+      columnNumber: lineColumn.columnNumber - 1,
+      lineNumber: lineColumn.lineNumber - 1,
+    };
 
     const previous = this.hasSetOnLocationByRegexp(urlRegex, lineColumn);
     if (previous) {

src/adapter/console/textualMessage.ts

@@ -10,7 +10,7 @@ import Dap from '../../dap/api';
 import { formatMessage } from '../messageFormat';
 import { messageFormatters, previewAsObject } from '../objectPreview';
 import { AnyObject } from '../objectPreview/betterTypes';
-import { IUiLocation } from '../sources';
+import { IUiLocation } from '../source';
 import { StackFrame, StackTrace } from '../stackTrace';
 import { Thread } from '../threads';
 import { IConsoleMessage } from './consoleMessage';

src/adapter/debugAdapter.ts

@@ -36,7 +36,8 @@ import { BasicCpuProfiler } from './profiling/basicCpuProfiler';
 import { ScriptSkipper } from './scriptSkipper/implementation';
 import { IScriptSkipper } from './scriptSkipper/scriptSkipper';
 import { SmartStepper } from './smartStepping';
-import { ISourceWithMap, SourceContainer, SourceFromMap } from './sources';
+import { ISourceWithMap, SourceFromMap } from './source';
+import { SourceContainer } from './sourceContainer';
 import { Thread } from './threads';
 import { VariableStore } from './variableStore';
 
@@ -509,7 +510,7 @@ export class DebugAdapter implements IDisposable {
   async _prettyPrintSource(
     params: Dap.PrettyPrintSourceParams,
   ): Promise<Dap.PrettyPrintSourceResult | Dap.Error> {
-    if (!params.source) {
+    if (!params.source || !this._thread) {
       return { canPrettyPrint: false };
     }
 
@@ -526,7 +527,7 @@ export class DebugAdapter implements IDisposable {
 
     const { map: sourceMap, source: generated } = prettified;
 
-    this.breakpointManager.moveBreakpoints(source, sourceMap, generated);
+    await this.breakpointManager.moveBreakpoints(this._thread, source, sourceMap, generated);
     this.sourceContainer.clearDisabledSourceMaps(source as ISourceWithMap);
     await this._refreshStackTrace();
 

src/adapter/diagnosics.ts

@@ -18,7 +18,8 @@ import {
   IBreakpointCdpReferenceApplied,
   IBreakpointCdpReferencePending,
 } from './breakpoints/breakpointBase';
-import { IUiLocation, SourceContainer, SourceFromMap } from './sources';
+import { IUiLocation, SourceFromMap, isSourceWithSourceMap } from './source';
+import { SourceContainer } from './sourceContainer';
 
 export interface IDiagnosticSource {
   uniqueId: number;
@@ -148,14 +149,16 @@ export class Diagnostics {
                   ([k, v]) => [k.sourceReference, v] as [number, string],
                 )
               : undefined,
-          sourceMap: source.sourceMap && {
-            url: source.sourceMap.url,
-            metadata: source.sourceMap.metadata,
-            sources: mapValues(
-              Object.fromEntries(source.sourceMap.sourceByUrl),
-              v => v.sourceReference,
-            ),
-          },
+          sourceMap: isSourceWithSourceMap(source)
+            ? {
+                url: source.sourceMap.metadata.sourceMapUrl,
+                metadata: source.sourceMap.metadata,
+                sources: mapValues(
+                  Object.fromEntries(source.sourceMap.sourceByUrl),
+                  v => v.sourceReference,
+                ),
+              }
+            : undefined,
         }))(),
       );
     }

src/adapter/exceptionPauseService.test.ts

@@ -5,16 +5,16 @@
 import { expect } from 'chai';
 import { SinonStub, stub } from 'sinon';
 import Cdp from '../cdp/api';
-import { stubbedCdpApi, StubCdpApi } from '../cdp/stubbedApi';
+import { StubCdpApi, stubbedCdpApi } from '../cdp/stubbedApi';
 import { upcastPartial } from '../common/objUtils';
 import { AnyLaunchConfiguration } from '../configuration';
 import Dap from '../dap/api';
-import { stubbedDapApi, StubDapApi } from '../dap/stubbedApi';
+import { StubDapApi, stubbedDapApi } from '../dap/stubbedApi';
 import { assertNotResolved, assertResolved } from '../test/asserts';
 import { IEvaluator } from './evaluator';
 import { ExceptionPauseService, PauseOnExceptionsState } from './exceptionPauseService';
 import { ScriptSkipper } from './scriptSkipper/implementation';
-import { SourceContainer } from './sources';
+import { SourceContainer } from './sourceContainer';
 
 describe('ExceptionPauseService', () => {
   let prepareEval: SinonStub;