chore: use vscode-markdown-languageservice for link linting (#36901)

* chore: use vscode-markdown-languageservice for docs link linting

* docs: make links relative

docs/api/browser-window.md

@@ -659,9 +659,9 @@ Emitted when scroll wheel event phase has begun.
 
 > **Note**
 > This event is deprecated beginning in Electron 22.0.0. See [Breaking
-> Changes](breaking-changes.md#deprecated-browserwindow-scroll-touch--events)
+> Changes](../breaking-changes.md#deprecated-browserwindow-scroll-touch--events)
 > for details of how to migrate to using the [WebContents
-> `input-event`](api/web-contents.md#event-input-event) event.
+> `input-event`](./web-contents.md#event-input-event) event.
 
 #### Event: 'scroll-touch-end' _macOS_ _Deprecated_
 
@@ -669,9 +669,9 @@ Emitted when scroll wheel event phase has ended.
 
 > **Note**
 > This event is deprecated beginning in Electron 22.0.0. See [Breaking
-> Changes](breaking-changes.md#deprecated-browserwindow-scroll-touch--events)
+> Changes](../breaking-changes.md#deprecated-browserwindow-scroll-touch--events)
 > for details of how to migrate to using the [WebContents
-> `input-event`](api/web-contents.md#event-input-event) event.
+> `input-event`](./web-contents.md#event-input-event) event.
 
 #### Event: 'scroll-touch-edge' _macOS_ _Deprecated_
 
@@ -679,9 +679,9 @@ Emitted when scroll wheel event phase filed upon reaching the edge of element.
 
 > **Note**
 > This event is deprecated beginning in Electron 22.0.0. See [Breaking
-> Changes](breaking-changes.md#deprecated-browserwindow-scroll-touch--events)
+> Changes](../breaking-changes.md#deprecated-browserwindow-scroll-touch--events)
 > for details of how to migrate to using the [WebContents
-> `input-event`](api/web-contents.md#event-input-event) event.
+> `input-event`](./web-contents.md#event-input-event) event.
 
 #### Event: 'swipe' _macOS_
 

docs/breaking-changes.md

@@ -1433,7 +1433,7 @@ When building native modules for windows, the `win_delay_load_hook` variable in
 the module's `binding.gyp` must be true (which is the default). If this hook is
 not present, then the native module will fail to load on Windows, with an error
 message like `Cannot find module`. See the [native module
-guide](/docs/tutorial/using-native-node-modules.md) for more.
+guide](./tutorial/using-native-node-modules.md) for more.
 
 ### Removed: IA32 Linux support
 

package.json

@@ -5,6 +5,7 @@
   "description": "Build cross platform desktop apps with JavaScript, HTML, and CSS",
   "devDependencies": {
     "@azure/storage-blob": "^12.9.0",
+    "@dsanders11/vscode-markdown-languageservice": "^0.3.0-alpha.4",
     "@electron/asar": "^3.2.1",
     "@electron/docs-parser": "^1.0.0",
     "@electron/fiddle-core": "^1.0.4",
@@ -56,6 +57,7 @@
     "lint": "^1.1.2",
     "lint-staged": "^10.2.11",
     "markdownlint-cli": "^0.33.0",
+    "mdast-util-from-markdown": "^1.2.0",
     "minimist": "^1.2.6",
     "null-loader": "^4.0.0",
     "pre-flight": "^1.1.0",
@@ -72,6 +74,10 @@
     "ts-loader": "^8.0.2",
     "ts-node": "6.2.0",
     "typescript": "^4.5.5",
+    "unist-util-visit": "^4.1.1",
+    "vscode-languageserver": "^8.0.2",
+    "vscode-languageserver-textdocument": "^1.0.7",
+    "vscode-uri": "^3.0.6",
     "webpack": "^5.73.0",
     "webpack-cli": "^4.10.0",
     "wrapper-webpack-plugin": "^2.2.0"
@@ -89,7 +95,7 @@
     "lint:py": "node ./script/lint.js --py",
     "lint:gn": "node ./script/lint.js --gn",
     "lint:docs": "remark docs -qf && npm run lint:js-in-markdown && npm run create-typescript-definitions && npm run lint:docs-relative-links && npm run lint:markdownlint",
-    "lint:docs-relative-links": "python3 ./script/check-relative-doc-links.py",
+    "lint:docs-relative-links": "ts-node script/lint-docs-links.ts",
     "lint:markdownlint": "markdownlint -r ./script/markdownlint-emd001.js \"*.md\" \"docs/**/*.md\"",
     "lint:js-in-markdown": "standard-markdown docs",
     "create-api-json": "node script/create-api-json.js",

script/check-relative-doc-links.py

@@ -1,130 +0,0 @@
-#!/usr/bin/env python3
-
-from __future__ import print_function
-import os
-import sys
-import re
-
-
-SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
-DOCS_DIR = os.path.join(SOURCE_ROOT, 'docs')
-
-
-def main():
-  os.chdir(SOURCE_ROOT)
-
-  filepaths = []
-  totalDirs = 0
-  try:
-    for root, dirs, files in os.walk(DOCS_DIR):
-      totalDirs += len(dirs)
-      for f in files:
-        if f.endswith('.md'):
-          filepaths.append(os.path.join(root, f))
-  except KeyboardInterrupt:
-    print('Keyboard interruption. Please try again.')
-    return 0
-
-  totalBrokenLinks = 0
-  for path in filepaths:
-    totalBrokenLinks += getBrokenLinks(path)
-
-  print('Parsed through ' + str(len(filepaths)) +
-        ' files within docs directory and its ' +
-        str(totalDirs) + ' subdirectories.')
-  print('Found ' + str(totalBrokenLinks) + ' broken relative links.')
-  return totalBrokenLinks
-
-
-def getBrokenLinks(filepath):
-  currentDir = os.path.dirname(filepath)
-  brokenLinks = []
-
-  try:
-    f = open(filepath, 'r', encoding="utf-8")
-    lines = f.readlines()
-  except KeyboardInterrupt:
-    print('Keyboard interruption while parsing. Please try again.')
-  finally:
-    f.close()
-
-  linkRegexLink = re.compile('\[(.*?)\]\((?P<link>(.*?))\)')
-  referenceLinkRegex = re.compile(
-      '^\s{0,3}\[.*?\]:\s*(?P<link>[^<\s]+|<[^<>\r\n]+>)'
-  )
-  links = []
-  for line in lines:
-    matchLinks = linkRegexLink.search(line)
-    matchReferenceLinks = referenceLinkRegex.search(line)
-    if matchLinks:
-      relativeLink = matchLinks.group('link')
-      if not str(relativeLink).startswith('http'):
-        links.append(relativeLink)
-    if matchReferenceLinks:
-      referenceLink = matchReferenceLinks.group('link').strip('<>')
-      if not str(referenceLink).startswith('http'):
-        links.append(referenceLink)
-
-  for link in links:
-    sections = link.split('#')
-    if len(sections) < 2:
-      if not os.path.isfile(os.path.join(currentDir, link)):
-        brokenLinks.append(link)
-    elif str(link).startswith('#'):
-      if not checkSections(sections, lines):
-        brokenLinks.append(link)
-    else:
-      tempFile = os.path.join(currentDir, sections[0])
-      if os.path.isfile(tempFile):
-        try:
-          newFile = open(tempFile, 'r', encoding="utf-8")
-          newLines = newFile.readlines()
-        except KeyboardInterrupt:
-          print('Keyboard interruption while parsing. Please try again.')
-        finally:
-          newFile.close()
-
-        if not checkSections(sections, newLines):
-          brokenLinks.append(link)
-      else:
-        brokenLinks.append(link)
-
-
-  print_errors(filepath, brokenLinks)
-  return len(brokenLinks)
-
-
-def checkSections(sections, lines):
-  invalidCharsRegex = '[^A-Za-z0-9_ \-]'
-  sectionHeader = sections[1]
-  regexSectionTitle = re.compile('# (?P<header>.*)')
-  for line in lines:
-    matchHeader = regexSectionTitle.search(line)
-    if matchHeader:
-      # This does the following to slugify a header name:
-      #  * Replace whitespace with dashes
-      #  * Strip anything that's not alphanumeric or a dash
-      #  * Anything quoted with backticks (`) is an exception and will
-      #    not have underscores stripped
-      matchHeader = str(matchHeader.group('header')).replace(' ', '-')
-      matchHeader = ''.join(
-        map(
-          lambda match: re.sub(invalidCharsRegex, '', match[0])
-          + re.sub(invalidCharsRegex + '|_', '', match[1]),
-          re.findall('(`[^`]+`)|([^`]+)', matchHeader),
-        )
-      )
-      if matchHeader.lower() == sectionHeader:
-        return True
-  return False
-
-
-def print_errors(filepath, brokenLink):
-  if brokenLink:
-    print("File Location: " + filepath)
-    for link in brokenLink:
-      print("\tBroken links: " + link)
-
-
-if __name__ == '__main__':
-  sys.exit(main())

script/lib/markdown.ts

@@ -0,0 +1,334 @@
+import * as fs from 'fs';
+import * as path from 'path';
+
+import * as MarkdownIt from 'markdown-it';
+import {
+  githubSlugifier,
+  resolveInternalDocumentLink,
+  ExternalHref,
+  FileStat,
+  HrefKind,
+  InternalHref,
+  IMdLinkComputer,
+  IMdParser,
+  ITextDocument,
+  IWorkspace,
+  MdLink,
+  MdLinkKind
+} from '@dsanders11/vscode-markdown-languageservice';
+import { Emitter, Range } from 'vscode-languageserver';
+import { TextDocument } from 'vscode-languageserver-textdocument';
+import { URI } from 'vscode-uri';
+
+import { findMatchingFiles } from './utils';
+
+import type { Definition, ImageReference, Link, LinkReference } from 'mdast';
+import type { fromMarkdown as FromMarkdownFunction } from 'mdast-util-from-markdown';
+import type { Node, Position } from 'unist';
+import type { visit as VisitFunction } from 'unist-util-visit';
+
+// Helper function to work around import issues with ESM modules and ts-node
+// eslint-disable-next-line no-new-func
+const dynamicImport = new Function('specifier', 'return import(specifier)');
+
+// Helper function from `vscode-markdown-languageservice` codebase
+function tryDecodeUri (str: string): string {
+  try {
+    return decodeURI(str);
+  } catch {
+    return str;
+  }
+}
+
+// Helper function from `vscode-markdown-languageservice` codebase
+function createHref (
+  sourceDocUri: URI,
+  link: string,
+  workspace: IWorkspace
+): ExternalHref | InternalHref | undefined {
+  if (/^[a-z-][a-z-]+:/i.test(link)) {
+    // Looks like a uri
+    return { kind: HrefKind.External, uri: URI.parse(tryDecodeUri(link)) };
+  }
+
+  const resolved = resolveInternalDocumentLink(sourceDocUri, link, workspace);
+  if (!resolved) {
+    return undefined;
+  }
+
+  return {
+    kind: HrefKind.Internal,
+    path: resolved.resource,
+    fragment: resolved.linkFragment
+  };
+}
+
+function positionToRange (position: Position): Range {
+  return {
+    start: {
+      character: position.start.column - 1,
+      line: position.start.line - 1
+    },
+    end: { character: position.end.column - 1, line: position.end.line - 1 }
+  };
+}
+
+const mdIt = MarkdownIt({ html: true });
+
+export class MarkdownParser implements IMdParser {
+  slugifier = githubSlugifier;
+
+  async tokenize (document: TextDocument) {
+    return mdIt.parse(document.getText(), {});
+  }
+}
+
+export class DocsWorkspace implements IWorkspace {
+  private readonly documentCache: Map<string, TextDocument>;
+  readonly root: string;
+
+  constructor (root: string) {
+    this.documentCache = new Map();
+    this.root = root;
+  }
+
+  get workspaceFolders () {
+    return [URI.file(this.root)];
+  }
+
+  async getAllMarkdownDocuments (): Promise<Iterable<ITextDocument>> {
+    const files = await findMatchingFiles(this.root, (file) =>
+      file.endsWith('.md')
+    );
+
+    for (const file of files) {
+      const document = TextDocument.create(
+        URI.file(file).toString(),
+        'markdown',
+        1,
+        fs.readFileSync(file, 'utf8')
+      );
+
+      this.documentCache.set(file, document);
+    }
+
+    return this.documentCache.values();
+  }
+
+  hasMarkdownDocument (resource: URI) {
+    const relativePath = path.relative(this.root, resource.path);
+    return (
+      !relativePath.startsWith('..') &&
+      !path.isAbsolute(relativePath) &&
+      fs.existsSync(resource.path)
+    );
+  }
+
+  async openMarkdownDocument (resource: URI) {
+    if (!this.documentCache.has(resource.path)) {
+      const document = TextDocument.create(
+        resource.toString(),
+        'markdown',
+        1,
+        fs.readFileSync(resource.path, 'utf8')
+      );
+
+      this.documentCache.set(resource.path, document);
+    }
+
+    return this.documentCache.get(resource.path);
+  }
+
+  async stat (resource: URI): Promise<FileStat | undefined> {
+    if (this.hasMarkdownDocument(resource)) {
+      const stats = fs.statSync(resource.path);
+      return { isDirectory: stats.isDirectory() };
+    }
+
+    return undefined;
+  }
+
+  async readDirectory (): Promise<Iterable<readonly [string, FileStat]>> {
+    throw new Error('Not implemented');
+  }
+
+  //
+  // These events are defined to fulfill the interface, but are never emitted
+  // by this implementation since it's not meant for watching a workspace
+  //
+
+  #onDidChangeMarkdownDocument = new Emitter<ITextDocument>();
+  onDidChangeMarkdownDocument = this.#onDidChangeMarkdownDocument.event;
+
+  #onDidCreateMarkdownDocument = new Emitter<ITextDocument>();
+  onDidCreateMarkdownDocument = this.#onDidCreateMarkdownDocument.event;
+
+  #onDidDeleteMarkdownDocument = new Emitter<URI>();
+  onDidDeleteMarkdownDocument = this.#onDidDeleteMarkdownDocument.event;
+}
+
+export class MarkdownLinkComputer implements IMdLinkComputer {
+  private readonly workspace: IWorkspace;
+
+  constructor (workspace: IWorkspace) {
+    this.workspace = workspace;
+  }
+
+  async getAllLinks (document: ITextDocument): Promise<MdLink[]> {
+    const { fromMarkdown } = (await dynamicImport(
+      'mdast-util-from-markdown'
+    )) as { fromMarkdown: typeof FromMarkdownFunction };
+
+    const tree = fromMarkdown(document.getText());
+
+    const links = [
+      ...(await this.#getInlineLinks(document, tree)),
+      ...(await this.#getReferenceLinks(document, tree)),
+      ...(await this.#getLinkDefinitions(document, tree))
+    ];
+
+    return links;
+  }
+
+  async #getInlineLinks (
+    document: ITextDocument,
+    tree: Node
+  ): Promise<MdLink[]> {
+    const { visit } = (await dynamicImport('unist-util-visit')) as {
+      visit: typeof VisitFunction;
+    };
+
+    const documentUri = URI.parse(document.uri);
+    const links: MdLink[] = [];
+
+    visit(
+      tree,
+      (node) => node.type === 'link',
+      (node: Node) => {
+        const link = node as Link;
+        const href = createHref(documentUri, link.url, this.workspace);
+
+        if (href) {
+          const range = positionToRange(link.position!);
+
+          // NOTE - These haven't been implemented properly, but their
+          //        values aren't used for the link linting use-case
+          const targetRange = range;
+          const hrefRange = range;
+          const fragmentRange = undefined;
+
+          links.push({
+            kind: MdLinkKind.Link,
+            href,
+            source: {
+              hrefText: link.url,
+              resource: documentUri,
+              range,
+              targetRange,
+              hrefRange,
+              fragmentRange,
+              pathText: link.url.split('#')[0]
+            }
+          });
+        }
+      }
+    );
+
+    return links;
+  }
+
+  async #getReferenceLinks (
+    document: ITextDocument,
+    tree: Node
+  ): Promise<MdLink[]> {
+    const { visit } = (await dynamicImport('unist-util-visit')) as {
+      visit: typeof VisitFunction;
+    };
+
+    const links: MdLink[] = [];
+
+    visit(
+      tree,
+      (node) => ['imageReference', 'linkReference'].includes(node.type),
+      (node: Node) => {
+        const link = node as ImageReference | LinkReference;
+        const range = positionToRange(link.position!);
+
+        // NOTE - These haven't been implemented properly, but their
+        //        values aren't used for the link linting use-case
+        const targetRange = range;
+        const hrefRange = range;
+
+        links.push({
+          kind: MdLinkKind.Link,
+          href: {
+            kind: HrefKind.Reference,
+            ref: link.label!
+          },
+          source: {
+            hrefText: link.label!,
+            resource: URI.parse(document.uri),
+            range,
+            targetRange,
+            hrefRange,
+            fragmentRange: undefined,
+            pathText: link.label!
+          }
+        });
+      }
+    );
+
+    return links;
+  }
+
+  async #getLinkDefinitions (
+    document: ITextDocument,
+    tree: Node
+  ): Promise<MdLink[]> {
+    const { visit } = (await dynamicImport('unist-util-visit')) as {
+      visit: typeof VisitFunction;
+    };
+
+    const documentUri = URI.parse(document.uri);
+    const links: MdLink[] = [];
+
+    visit(
+      tree,
+      (node) => node.type === 'definition',
+      (node: Node) => {
+        const definition = node as Definition;
+        const href = createHref(documentUri, definition.url, this.workspace);
+
+        if (href) {
+          const range = positionToRange(definition.position!);
+
+          // NOTE - These haven't been implemented properly, but their
+          //        values aren't used for the link linting use-case
+          const targetRange = range;
+          const hrefRange = range;
+          const fragmentRange = undefined;
+
+          links.push({
+            kind: MdLinkKind.Definition,
+            href,
+            ref: {
+              range,
+              text: definition.label!
+            },
+            source: {
+              hrefText: definition.url,
+              resource: documentUri,
+              range,
+              targetRange,
+              hrefRange,
+              fragmentRange,
+              pathText: definition.url.split('#')[0]
+            }
+          });
+        }
+      }
+    );
+
+    return links;
+  }
+}

script/lib/utils.js

@@ -1,5 +1,6 @@
 const { GitProcess } = require('dugite');
 const fs = require('fs');
+const klaw = require('klaw');
 const os = require('os');
 const path = require('path');
 
@@ -122,8 +123,29 @@ function chunkFilenames (filenames, offset = 0) {
   );
 }
 
+/**
+ * @param {string} top
+ * @param {(filename: string) => boolean} test
+ * @returns {Promise<string[]>}
+*/
+async function findMatchingFiles (top, test) {
+  return new Promise((resolve, reject) => {
+    const matches = [];
+    klaw(top, {
+      filter: f => path.basename(f) !== '.bin'
+    })
+      .on('end', () => resolve(matches))
+      .on('data', item => {
+        if (test(item.path)) {
+          matches.push(item.path);
+        }
+      });
+  });
+}
+
 module.exports = {
   chunkFilenames,
+  findMatchingFiles,
   getCurrentBranch,
   getElectronExec,
   getOutDir,

script/lint-docs-links.ts

@@ -0,0 +1,177 @@
+#!/usr/bin/env ts-node
+
+import * as path from 'path';
+
+import {
+  createLanguageService,
+  DiagnosticLevel,
+  DiagnosticOptions,
+  ILogger
+} from '@dsanders11/vscode-markdown-languageservice';
+import * as minimist from 'minimist';
+import fetch from 'node-fetch';
+import { CancellationTokenSource } from 'vscode-languageserver';
+import { URI } from 'vscode-uri';
+
+import {
+  DocsWorkspace,
+  MarkdownLinkComputer,
+  MarkdownParser
+} from './lib/markdown';
+
+class NoOpLogger implements ILogger {
+  log (): void {}
+}
+
+const diagnosticOptions: DiagnosticOptions = {
+  ignoreLinks: [],
+  validateDuplicateLinkDefinitions: DiagnosticLevel.error,
+  validateFileLinks: DiagnosticLevel.error,
+  validateFragmentLinks: DiagnosticLevel.error,
+  validateMarkdownFileLinkFragments: DiagnosticLevel.error,
+  validateReferences: DiagnosticLevel.error,
+  validateUnusedLinkDefinitions: DiagnosticLevel.error
+};
+
+async function fetchExternalLink (link: string, checkRedirects = false) {
+  try {
+    const response = await fetch(link);
+    if (response.status !== 200) {
+      console.log('Broken link', link, response.status, response.statusText);
+    } else {
+      if (checkRedirects && response.redirected) {
+        const wwwUrl = new URL(link);
+        wwwUrl.hostname = `www.${wwwUrl.hostname}`;
+
+        // For now cut down on noise to find meaningful redirects
+        const wwwRedirect = wwwUrl.toString() === response.url;
+        const trailingSlashRedirect = `${link}/` === response.url;
+
+        if (!wwwRedirect && !trailingSlashRedirect) {
+          console.log('Link redirection', link, '->', response.url);
+        }
+      }
+
+      return true;
+    }
+  } catch {
+    console.log('Broken link', link);
+  }
+
+  return false;
+}
+
+async function main ({ fetchExternalLinks = false, checkRedirects = false }) {
+  const workspace = new DocsWorkspace(path.resolve(__dirname, '..', 'docs'));
+  const parser = new MarkdownParser();
+  const linkComputer = new MarkdownLinkComputer(workspace);
+  const languageService = createLanguageService({
+    workspace,
+    parser,
+    logger: new NoOpLogger(),
+    linkComputer
+  });
+
+  const cts = new CancellationTokenSource();
+  let errors = false;
+
+  const externalLinks = new Set<string>();
+
+  try {
+    // Collect diagnostics for all documents in the workspace
+    for (const document of await workspace.getAllMarkdownDocuments()) {
+      for (let link of await languageService.getDocumentLinks(
+        document,
+        cts.token
+      )) {
+        if (link.target === undefined) {
+          link =
+            (await languageService.resolveDocumentLink(link, cts.token)) ??
+            link;
+        }
+
+        if (
+          link.target &&
+          link.target.startsWith('http') &&
+          new URL(link.target).hostname !== 'localhost'
+        ) {
+          externalLinks.add(link.target);
+        }
+      }
+      const diagnostics = await languageService.computeDiagnostics(
+        document,
+        diagnosticOptions,
+        cts.token
+      );
+
+      if (diagnostics.length) {
+        console.log(
+          'File Location:',
+          path.relative(workspace.root, URI.parse(document.uri).path)
+        );
+      }
+
+      for (const diagnostic of diagnostics) {
+        console.log(
+          `\tBroken link on line ${diagnostic.range.start.line + 1}:`,
+          diagnostic.message
+        );
+        errors = true;
+      }
+    }
+  } finally {
+    cts.dispose();
+  }
+
+  if (fetchExternalLinks) {
+    const externalLinkStates = await Promise.all(
+      Array.from(externalLinks).map((link) =>
+        fetchExternalLink(link, checkRedirects)
+      )
+    );
+
+    errors = errors || !externalLinkStates.every((x) => x);
+  }
+
+  return errors;
+}
+
+function parseCommandLine () {
+  const showUsage = (arg?: string): boolean => {
+    if (!arg || arg.startsWith('-')) {
+      console.log(
+        'Usage: script/lint-docs-links.ts [-h|--help] [--fetch-external-links] ' +
+          '[--check-redirects]'
+      );
+      process.exit(0);
+    }
+
+    return true;
+  };
+
+  const opts = minimist(process.argv.slice(2), {
+    boolean: ['help', 'fetch-external-links', 'check-redirects'],
+    stopEarly: true,
+    unknown: showUsage
+  });
+
+  if (opts.help) showUsage();
+
+  return opts;
+}
+
+if (process.mainModule === module) {
+  const opts = parseCommandLine();
+
+  main({
+    fetchExternalLinks: opts['fetch-external-links'],
+    checkRedirects: opts['check-redirects']
+  })
+    .then((errors) => {
+      if (errors) process.exit(1);
+    })
+    .catch((error) => {
+      console.error(error);
+      process.exit(1);
+    });
+}

script/lint.js

@@ -5,11 +5,10 @@ const { GitProcess } = require('dugite');
 const childProcess = require('child_process');
 const { ESLint } = require('eslint');
 const fs = require('fs');
-const klaw = require('klaw');
 const minimist = require('minimist');
 const path = require('path');
 
-const { chunkFilenames } = require('./lib/utils');
+const { chunkFilenames, findMatchingFiles } = require('./lib/utils');
 
 const ELECTRON_ROOT = path.normalize(path.dirname(__dirname));
 const SOURCE_ROOT = path.resolve(ELECTRON_ROOT, '..');
@@ -279,21 +278,6 @@ async function findChangedFiles (top) {
   return new Set(absolutePaths);
 }
 
-async function findMatchingFiles (top, test) {
-  return new Promise((resolve, reject) => {
-    const matches = [];
-    klaw(top, {
-      filter: f => path.basename(f) !== '.bin'
-    })
-      .on('end', () => resolve(matches))
-      .on('data', item => {
-        if (test(item.path)) {
-          matches.push(item.path);
-        }
-      });
-  });
-}
-
 async function findFiles (args, linter) {
   let filenames = [];
   let includelist = null;

script/run-clang-tidy.ts

@@ -1,6 +1,5 @@
 import * as childProcess from 'child_process';
 import * as fs from 'fs';
-import * as klaw from 'klaw';
 import * as minimist from 'minimist';
 import * as os from 'os';
 import * as path from 'path';
@@ -9,7 +8,7 @@ import * as streamJson from 'stream-json';
 import { ignore as streamJsonIgnore } from 'stream-json/filters/Ignore';
 import { streamArray as streamJsonStreamArray } from 'stream-json/streamers/StreamArray';
 
-import { chunkFilenames } from './lib/utils';
+import { chunkFilenames, findMatchingFiles } from './lib/utils';
 
 const SOURCE_ROOT = path.normalize(path.dirname(__dirname));
 const LLVM_BIN = path.resolve(
@@ -204,24 +203,6 @@ async function runClangTidy (
   }
 }
 
-async function findMatchingFiles (
-  top: string,
-  test: (filename: string) => boolean
-): Promise<string[]> {
-  return new Promise((resolve) => {
-    const matches = [] as string[];
-    klaw(top, {
-      filter: (f) => path.basename(f) !== '.bin'
-    })
-      .on('end', () => resolve(matches))
-      .on('data', (item) => {
-        if (test(item.path)) {
-          matches.push(item.path);
-        }
-      });
-  });
-}
-
 function parseCommandLine () {
   const showUsage = (arg?: string) : boolean => {
     if (!arg || arg.startsWith('-')) {

yarn.lock

@@ -111,6 +111,17 @@
   resolved "https://registry.yarnpkg.com/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz#1d572bfbbe14b7704e0ba0f39b74815b84870d70"
   integrity sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==
 
+"@dsanders11/vscode-markdown-languageservice@^0.3.0-alpha.4":
+  version "0.3.0-alpha.4"
+  resolved "https://registry.yarnpkg.com/@dsanders11/vscode-markdown-languageservice/-/vscode-markdown-languageservice-0.3.0-alpha.4.tgz#cd80b82142a2c10e09b5f36a93c3ea65b7d2a7f9"
+  integrity sha512-MHp/CniEkzJb1CAw/bHwucuImaICrcIuohEFamTW8sJC2jhKCnbYblwJFZ3OOS3wTYZbzFIa8WZ4Dn5yL2E7jg==
+  dependencies:
+    "@vscode/l10n" "^0.0.10"
+    picomatch "^2.3.1"
+    vscode-languageserver-textdocument "^1.0.5"
+    vscode-languageserver-types "^3.17.1"
+    vscode-uri "^3.0.3"
+
 "@electron/asar@^3.2.1":
   version "3.2.1"
   resolved "https://registry.yarnpkg.com/@electron/asar/-/asar-3.2.1.tgz#c4143896f3dd43b59a80a9c9068d76f77efb62ea"
@@ -1169,6 +1180,11 @@
     "@typescript-eslint/types" "4.4.1"
     eslint-visitor-keys "^2.0.0"
 
+"@vscode/l10n@^0.0.10":
+  version "0.0.10"
+  resolved "https://registry.yarnpkg.com/@vscode/l10n/-/l10n-0.0.10.tgz#9c513107c690c0dd16e3ec61e453743de15ebdb0"
+  integrity sha512-E1OCmDcDWa0Ya7vtSjp/XfHFGqYJfh+YPC1RkATU71fTac+j1JjCcB3qwSzmlKAighx2WxhLlfhS0RwAN++PFQ==
+
 "@webassemblyjs/[email protected]":
   version "1.11.1"
   resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.11.1.tgz#2bfd767eae1a6996f432ff7e8d7fc75679c0b6a7"
@@ -2097,6 +2113,13 @@ debug@^4.1.0, debug@^4.3.3, debug@^4.3.4:
   dependencies:
     ms "2.1.2"
 
+decode-named-character-reference@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz#daabac9690874c394c81e4162a0304b35d824f0e"
+  integrity sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==
+  dependencies:
+    character-entities "^2.0.0"
+
 decompress-response@^3.3.0:
   version "3.3.0"
   resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-3.3.0.tgz#80a4dd323748384bfa248083622aedec982adff3"
@@ -2184,6 +2207,11 @@ deprecation@^2.0.0, deprecation@^2.3.1:
   resolved "https://registry.yarnpkg.com/deprecation/-/deprecation-2.3.1.tgz#6368cbdb40abf3373b525ac87e4a260c3a700919"
   integrity sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==
 
+dequal@^2.0.0:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/dequal/-/dequal-2.0.3.tgz#2644214f1997d39ed0ee0ece72335490a7ac67be"
+  integrity sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==
+
 destroy@~1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80"
@@ -2199,6 +2227,11 @@ diff@^3.1.0:
   resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12"
   integrity sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==
 
+diff@^5.0.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/diff/-/diff-5.1.0.tgz#bc52d298c5ea8df9194800224445ed43ffc87e40"
+  integrity sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==
+
 dir-glob@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f"
@@ -3917,6 +3950,11 @@ klaw@^3.0.0:
   dependencies:
     graceful-fs "^4.1.9"
 
+kleur@^4.0.3:
+  version "4.1.5"
+  resolved "https://registry.yarnpkg.com/kleur/-/kleur-4.1.5.tgz#95106101795f7050c6c650f350c683febddb1780"
+  integrity sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==
+
 levn@^0.3.0, levn@~0.3.0:
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee"
@@ -4275,6 +4313,24 @@ mdast-util-from-markdown@^1.0.0:
     parse-entities "^3.0.0"
     unist-util-stringify-position "^3.0.0"
 
+mdast-util-from-markdown@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/mdast-util-from-markdown/-/mdast-util-from-markdown-1.2.0.tgz#84df2924ccc6c995dec1e2368b2b208ad0a76268"
+  integrity sha512-iZJyyvKD1+K7QX1b5jXdE7Sc5dtoTry1vzV28UZZe8Z1xVnB/czKntJ7ZAkG0tANqRnBF6p3p7GpU1y19DTf2Q==
+  dependencies:
+    "@types/mdast" "^3.0.0"
+    "@types/unist" "^2.0.0"
+    decode-named-character-reference "^1.0.0"
+    mdast-util-to-string "^3.1.0"
+    micromark "^3.0.0"
+    micromark-util-decode-numeric-character-reference "^1.0.0"
+    micromark-util-decode-string "^1.0.0"
+    micromark-util-normalize-identifier "^1.0.0"
+    micromark-util-symbol "^1.0.0"
+    micromark-util-types "^1.0.0"
+    unist-util-stringify-position "^3.0.0"
+    uvu "^0.5.0"
+
 mdast-util-heading-style@^1.0.2:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/mdast-util-heading-style/-/mdast-util-heading-style-1.0.5.tgz#81b2e60d76754198687db0e8f044e42376db0426"
@@ -4297,7 +4353,7 @@ mdast-util-to-string@^1.0.2:
   resolved "https://registry.yarnpkg.com/mdast-util-to-string/-/mdast-util-to-string-1.0.6.tgz#7d85421021343b33de1552fc71cb8e5b4ae7536d"
   integrity sha512-868pp48gUPmZIhfKrLbaDneuzGiw3OTDjHc5M1kAepR2CWBJ+HpEsm252K4aXdiP5coVZaJPOqGtVU6Po8xnXg==
 
-mdast-util-to-string@^3.0.0:
+mdast-util-to-string@^3.0.0, mdast-util-to-string@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/mdast-util-to-string/-/mdast-util-to-string-3.1.0.tgz#56c506d065fbf769515235e577b5a261552d56e9"
   integrity sha512-n4Vypz/DZgwo0iMHLQL49dJzlp7YtAJP+N07MZHpjPf/5XJuHUWstviF4Mn2jEiR/GNmtnRRqnwsXExk3igfFA==
@@ -4446,6 +4502,16 @@ micromark-util-decode-numeric-character-reference@^1.0.0:
   dependencies:
     micromark-util-symbol "^1.0.0"
 
+micromark-util-decode-string@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/micromark-util-decode-string/-/micromark-util-decode-string-1.0.2.tgz#942252ab7a76dec2dbf089cc32505ee2bc3acf02"
+  integrity sha512-DLT5Ho02qr6QWVNYbRZ3RYOSSWWFuH3tJexd3dgN1odEuPNxCngTCXJum7+ViRAd9BbdxCvMToPOD/IvVhzG6Q==
+  dependencies:
+    decode-named-character-reference "^1.0.0"
+    micromark-util-character "^1.0.0"
+    micromark-util-decode-numeric-character-reference "^1.0.0"
+    micromark-util-symbol "^1.0.0"
+
 micromark-util-encode@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/micromark-util-encode/-/micromark-util-encode-1.0.0.tgz#c409ecf751a28aa9564b599db35640fccec4c068"
@@ -4630,6 +4696,11 @@ mkdirp@^0.5.1, mkdirp@^0.5.5:
   dependencies:
     minimist "^1.2.5"
 
+mri@^1.1.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/mri/-/mri-1.2.0.tgz#6721480fec2a11a4889861115a48b6cbe7cc8f0b"
+  integrity sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==
+
 [email protected]:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
@@ -5080,6 +5151,11 @@ picomatch@^2.2.1:
   resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.2.2.tgz#21f333e9b6b8eaff02468f5146ea406d345f4dad"
   integrity sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==
 
+picomatch@^2.3.1:
+  version "2.3.1"
+  resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42"
+  integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==
+
 pify@^2.0.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c"
@@ -6034,6 +6110,13 @@ rxjs@^6.5.5, rxjs@^6.6.0:
   dependencies:
     tslib "^1.9.0"
 
+sade@^1.7.3:
+  version "1.8.1"
+  resolved "https://registry.yarnpkg.com/sade/-/sade-1.8.1.tgz#0a78e81d658d394887be57d2a409bf703a3b2701"
+  integrity sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==
+  dependencies:
+    mri "^1.1.0"
+
 [email protected], safe-buffer@~5.1.0, safe-buffer@~5.1.1:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
@@ -6911,6 +6994,11 @@ unist-util-is@^4.0.0:
   resolved "https://registry.yarnpkg.com/unist-util-is/-/unist-util-is-4.1.0.tgz#976e5f462a7a5de73d94b706bac1b90671b57797"
   integrity sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==
 
+unist-util-is@^5.0.0:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/unist-util-is/-/unist-util-is-5.1.1.tgz#e8aece0b102fa9bc097b0fef8f870c496d4a6236"
+  integrity sha512-F5CZ68eYzuSvJjGhCLPL3cYx45IxkqXSetCcRgUXtbcm50X2L9oOWQlfUfDdAf+6Pd27YDblBfdtmsThXmwpbQ==
+
 unist-util-position@^3.0.0:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/unist-util-position/-/unist-util-position-3.0.3.tgz#fff942b879538b242096c148153826664b1ca373"
@@ -6938,6 +7026,14 @@ unist-util-visit-parents@^3.0.0:
     "@types/unist" "^2.0.0"
     unist-util-is "^4.0.0"
 
+unist-util-visit-parents@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/unist-util-visit-parents/-/unist-util-visit-parents-5.1.1.tgz#868f353e6fce6bf8fa875b251b0f4fec3be709bb"
+  integrity sha512-gks4baapT/kNRaWxuGkl5BIhoanZo7sC/cUT/JToSRNL1dYoXRFl75d++NkjYk4TAu2uv2Px+l8guMajogeuiw==
+  dependencies:
+    "@types/unist" "^2.0.0"
+    unist-util-is "^5.0.0"
+
 unist-util-visit@^2.0.0:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-2.0.3.tgz#c3703893146df47203bb8a9795af47d7b971208c"
@@ -6947,6 +7043,15 @@ unist-util-visit@^2.0.0:
     unist-util-is "^4.0.0"
     unist-util-visit-parents "^3.0.0"
 
+unist-util-visit@^4.1.1:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-4.1.1.tgz#1c4842d70bd3df6cc545276f5164f933390a9aad"
+  integrity sha512-n9KN3WV9k4h1DxYR1LoajgN93wpEi/7ZplVe02IoB4gH5ctI1AaF2670BLHQYbwj+pY83gFtyeySFiyMHJklrg==
+  dependencies:
+    "@types/unist" "^2.0.0"
+    unist-util-is "^5.0.0"
+    unist-util-visit-parents "^5.1.1"
+
 universal-github-app-jwt@^1.0.1:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/universal-github-app-jwt/-/universal-github-app-jwt-1.1.0.tgz#0abaa876101cdf1d3e4c546be2768841c0c1b514"
@@ -7030,6 +7135,16 @@ uuid@^8.3.0:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
   integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
 
+uvu@^0.5.0:
+  version "0.5.6"
+  resolved "https://registry.yarnpkg.com/uvu/-/uvu-0.5.6.tgz#2754ca20bcb0bb59b64e9985e84d2e81058502df"
+  integrity sha512-+g8ENReyr8YsOc6fv/NVJs2vFdHBnBNdfE49rshrTzDWOlUx4Gq7KOS2GD8eqhy2j+Ejq29+SbKH8yjkAqXqoA==
+  dependencies:
+    dequal "^2.0.0"
+    diff "^5.0.0"
+    kleur "^4.0.3"
+    sade "^1.7.3"
+
 v8-compile-cache@^2.0.3:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz#54bc3cdd43317bca91e35dcaf305b1a7237de745"
@@ -7099,6 +7214,41 @@ vfile@^5.0.0:
     unist-util-stringify-position "^3.0.0"
     vfile-message "^3.0.0"
 
[email protected]:
+  version "8.0.2"
+  resolved "https://registry.yarnpkg.com/vscode-jsonrpc/-/vscode-jsonrpc-8.0.2.tgz#f239ed2cd6004021b6550af9fd9d3e47eee3cac9"
+  integrity sha512-RY7HwI/ydoC1Wwg4gJ3y6LpU9FJRZAUnTYMXthqhFXXu77ErDd/xkREpGuk4MyYkk4a+XDWAMqe0S3KkelYQEQ==
+
[email protected]:
+  version "3.17.2"
+  resolved "https://registry.yarnpkg.com/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.17.2.tgz#beaa46aea06ed061576586c5e11368a9afc1d378"
+  integrity sha512-8kYisQ3z/SQ2kyjlNeQxbkkTNmVFoQCqkmGrzLH6A9ecPlgTbp3wDTnUNqaUxYr4vlAcloxx8zwy7G5WdguYNg==
+  dependencies:
+    vscode-jsonrpc "8.0.2"
+    vscode-languageserver-types "3.17.2"
+
+vscode-languageserver-textdocument@^1.0.5, vscode-languageserver-textdocument@^1.0.7:
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.7.tgz#16df468d5c2606103c90554ae05f9f3d335b771b"
+  integrity sha512-bFJH7UQxlXT8kKeyiyu41r22jCZXG8kuuVVA33OEJn1diWOZK5n8zBSPZFHVBOu8kXZ6h0LIRhf5UnCo61J4Hg==
+
[email protected], vscode-languageserver-types@^3.17.1:
+  version "3.17.2"
+  resolved "https://registry.yarnpkg.com/vscode-languageserver-types/-/vscode-languageserver-types-3.17.2.tgz#b2c2e7de405ad3d73a883e91989b850170ffc4f2"
+  integrity sha512-zHhCWatviizPIq9B7Vh9uvrH6x3sK8itC84HkamnBWoDFJtzBf7SWlpLCZUit72b3os45h6RWQNC9xHRDF8dRA==
+
+vscode-languageserver@^8.0.2:
+  version "8.0.2"
+  resolved "https://registry.yarnpkg.com/vscode-languageserver/-/vscode-languageserver-8.0.2.tgz#cfe2f0996d9dfd40d3854e786b2821604dfec06d"
+  integrity sha512-bpEt2ggPxKzsAOZlXmCJ50bV7VrxwCS5BI4+egUmure/oI/t4OlFzi/YNtVvY24A2UDOZAgwFGgnZPwqSJubkA==
+  dependencies:
+    vscode-languageserver-protocol "3.17.2"
+
+vscode-uri@^3.0.3, vscode-uri@^3.0.6:
+  version "3.0.6"
+  resolved "https://registry.yarnpkg.com/vscode-uri/-/vscode-uri-3.0.6.tgz#5e6e2e1a4170543af30151b561a41f71db1d6f91"
+  integrity sha512-fmL7V1eiDBFRRnu+gfRWTzyPpNIHJTc4mWnFkwBUmO9U3KPgJAmTx7oxi2bl/Rh6HLdU7+4C9wlj0k2E4AdKFQ==
+
 walk-sync@^0.3.2:
   version "0.3.4"
   resolved "https://registry.yarnpkg.com/walk-sync/-/walk-sync-0.3.4.tgz#cf78486cc567d3a96b5b2237c6108017a5ffb9a4"