diff --git a/javascript/BUILD.bazel b/javascript/BUILD.bazel index 85b2024c283c..8f4a001265ae 100644 --- a/javascript/BUILD.bazel +++ b/javascript/BUILD.bazel @@ -33,5 +33,9 @@ codeql_pack( "//javascript/extractor:tools-extractor", "//javascript/resources", ], - zips = {"//javascript/extractor/lib/typescript": "tools"}, + arch_overrides = ["tools/typescript-parser-wrapper/node_modules/@typescript"], + zips = { + "//javascript/extractor/lib/typescript": "tools", + "//javascript/extractor/lib/typescript:native": "tools/typescript-parser-wrapper/node_modules/@typescript", + }, ) diff --git a/javascript/extractor/lib/typescript/BUILD.bazel b/javascript/extractor/lib/typescript/BUILD.bazel index 28003162673e..4d7c6edfd54f 100644 --- a/javascript/extractor/lib/typescript/BUILD.bazel +++ b/javascript/extractor/lib/typescript/BUILD.bazel @@ -1,12 +1,20 @@ -# Builds a zip file of the compiled typscript-parser-wrapper and its dependencies. +load("@rules_python//python:defs.bzl", "py_test") +load("//misc/bazel:os.bzl", "codeql_platform_select") +load("//misc/bazel:pkg.bzl", "codeql_pack", "codeql_pack_group") + +# Keep the JavaScript API common, but ship native executables in the arch package. genrule( - name = "typescript", + name = "build", srcs = [ "tsconfig.json", "package.json", "package-lock.json", + "package-runtime.mjs", ] + glob(["src/**"]), - outs = ["javascript.zip"], + outs = [ + "javascript.zip", + "native.zip", + ], cmd = "\n".join([ # the original working directory is not preserved anywhere, but needs to be accessible, as # all paths are relative to this @@ -25,20 +33,38 @@ genrule( "rm -rf $(RULEDIR)/inputs", "cp -L -R $$(dirname $(execpath package.json)) $(RULEDIR)/inputs", "cd $(RULEDIR)/inputs", - "$$NPM install", + "$$NPM ci", "$$NPM run build", "rm -rf node_modules", - # Install again with only runtime deps - "$$NPM install --prod", + "DISTRIBUTION=", + ]) + codeql_platform_select( + linux64 = "linux64", + linux_arm64 = "linux-arm64", + osx64 = "osx64", + win64 = "win64", + ) + "\n" + "\n".join([ + "$$NPM ci --omit=dev --omit=optional --ignore-scripts", + # Optional dependencies are host-filtered even with --force. Install the + # selected packages explicitly, retaining the lockfile's versions/integrities. + "$$NPM install --omit=dev --ignore-scripts --no-save --force $$(node package-runtime.mjs --packages $$DISTRIBUTION)", + "node package-runtime.mjs $$DISTRIBUTION", "mv node_modules build/", + # Node needs the ESM package boundary in the installed wrapper too. + "cp package.json build/", "mkdir -p typescript-parser-wrapper", "mv build/* typescript-parser-wrapper", - "OUT=$$BAZEL_ROOT/$@", + "OUT=$$BAZEL_ROOT/$(execpath javascript.zip)", + "NATIVE_OUT=$$BAZEL_ROOT/$(execpath native.zip)", "case $$OSTYPE in", - " cygwin|msys|win32) OUT=$$(cygpath -w $$OUT);;", + " cygwin|msys|win32) OUT=$$(cygpath -w $$OUT); NATIVE_OUT=$$(cygpath -w $$NATIVE_OUT);;", "esac", "", - "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$OUT $$(find typescript-parser-wrapper -name '*' -print)", + # Use response files to stay below Windows' command-line length limit. + "find typescript-parser-wrapper -print > wrapper-files", + "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$OUT @wrapper-files", + "cd native", + "find typescript-* -print > native-files", + "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$NATIVE_OUT @native-files", "rm -rf $$TEMP", ]), tools = [ @@ -48,3 +74,59 @@ genrule( ], visibility = ["//visibility:public"], ) + +filegroup( + name = "typescript", + srcs = ["javascript.zip"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "native", + srcs = ["native.zip"], + visibility = ["//visibility:public"], +) + +codeql_pack( + name = "package-test-pack", + srcs = [], + arch_overrides = ["tools/typescript-parser-wrapper/node_modules/@typescript"], + install_dest = None, + pack_prefix = "", + zips = { + ":native": "tools/typescript-parser-wrapper/node_modules/@typescript", + ":typescript": "tools", + }, +) + +codeql_pack_group( + name = "package-test-distribution", + srcs = [":package-test-pack"], + skip_installer = True, +) + +py_test( + name = "package-test", + size = "small", + srcs = ["test/package_test.py"], + data = [ + "package-runtime.mjs", + ":native", + ":package-test-distribution-arch-zip", + ":package-test-distribution-common-zip", + ":typescript", + "//misc/ripunzip", + "@nodejs//:node_bin", + ], + env = { + "NODE_BIN": "$(rlocationpath @nodejs//:node_bin)", + "PACKAGE_RUNTIME": "$(rlocationpath package-runtime.mjs)", + "RIPUNZIP": "$(rlocationpath //misc/ripunzip)", + "TS_ARCH_ZIP": "$(rlocationpath :package-test-distribution-arch-zip)", + "TS_COMMON_ZIP": "$(rlocationpath :package-test-distribution-common-zip)", + "TS_NATIVE_ZIP": "$(rlocationpath :native)", + "TS_WRAPPER_ZIP": "$(rlocationpath :typescript)", + }, + main = "test/package_test.py", + deps = ["@rules_python//python/runfiles"], +) diff --git a/javascript/extractor/lib/typescript/README.md b/javascript/extractor/lib/typescript/README.md index 998cec250aaa..fa765e134687 100644 --- a/javascript/extractor/lib/typescript/README.md +++ b/javascript/extractor/lib/typescript/README.md @@ -16,3 +16,229 @@ To debug the parser script: 2. Open VSCode and choose "Debug: Attach to Node process" from the command palette, and choose the process that looks like the parser-wrapper. + +Building and running +-------------------- + +The wrapper uses the exact `typescript@7.1.0-dev.20261015.1` package for both +compilation/type checking and the native runtime. Its integrity and those of all +platform-specific optional dependencies are recorded in `package-lock.json`. + +```sh +npm ci +npm run build +npm test +node build/main.js --version +``` + +Node.js 18.15.0, the existing Bazel toolchain version, supports the wrapper and +native API; no Node.js upgrade is required. Build scripts invoke TypeScript's +`lib/tsc.js` launcher explicitly because Node.js 18.15.0 cannot load its +extensionless ESM `bin/tsc` entry point. The installed wrapper must include its +`package.json` to retain the ESM package boundary. + +The existing TSLint rules require the legacy JavaScript compiler API, not the +native TypeScript 7 API. If compatible TSLint tooling is not already installed, +run it separately without replacing the wrapper's build/runtime dependency: + +```sh +npm exec --yes --package=tslint@6.1.3 --package=typescript@5.9.2 -- tslint --project . +``` + +`SEMMLE_TYPESCRIPT_HOME` still overrides the runtime dependency. It must point to +the installed TypeScript package directory (containing `package.json`), with the +native `typescript/sync` and `typescript/ast` entry points and matching executable. +A legacy 5.x module or a directory from which the bundled package merely happens +to be resolvable is not a valid override. + +Distribution packaging +---------------------- + +The Bazel build produces two archives: `:typescript` (`javascript.zip`) contains +the compiled ESM wrapper and JavaScript API; `:native` (`native.zip`) contains the +matching native packages for the target CodeQL distribution: + +| Distribution | Native packages | +| --- | --- | +| `linux64` | `@typescript/typescript-linux-x64` | +| `linux-arm64` | `@typescript/typescript-linux-arm64` | +| `osx64` | `@typescript/typescript-darwin-arm64` and `@typescript/typescript-darwin-x64` | +| `win64` | `@typescript/typescript-win32-x64` | + +The build installs the selected packages explicitly with `--force --no-save`, +using the pinned versions and integrities in the lockfile. Merely passing +`--force` to `npm ci` still filters optional dependencies by the build host. +`package-runtime.mjs` verifies the native package versions and marks executables +as `0755` before ZIP creation. The common CodeQL archive contains the wrapper; +the architecture archive installs native packages into +`tools/typescript-parser-wrapper/node_modules/@typescript`. Both archives must +be installed, with executable permissions preserved. No separate TypeScript +installation is needed. + +Run the installed-package smoke test with: + +```sh +bazel test //javascript/extractor/lib/typescript:package-test +``` + +It checks all distribution selections, mismatched runtime rejection, common/arch +ZIP separation and executable metadata, then installs the merged archives with +the production `ripunzip` and parses TS, TSX, and malformed source through the +installed JSON protocol. It runs on the host platform and uses Bazel's Node.js +runtime. The Java extractor test suite also installs both ZIPs and restores +executable permissions using their central-directory metadata. + +All four distribution runtime ZIPs have been cross-assembled and inspected. +Installed parsing has been verified with Node.js 18.15.0 on macOS arm64 and on +macOS x64 through Rosetta. CI integration tests have also passed on Linux x64, +Linux arm64, and Windows x64, including Windows 2022. + +Native lifecycle and memory +--------------------------- + +Each file owns a fresh native API/process. AST projection and JSON serialization +finish before the program is disposed and the API is closed, including on +failure. No native process or compiler cache is retained between files, during +stdout backpressure, or across reset/quit. This deliberately favors bounded +resource lifetime over reusing a compiler process; the measured cost is described +below. + +`SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD` remains a positive megabyte budget, defaulting +to 1000 when the Java extractor does not supply it. At native-process creation, +the wrapper subtracts Node's current RSS (not just its V8 heap) from that budget +and passes the remaining bytes, with a minimum of one, as the child's +`GOMEMLIMIT`. The parent's original `GOMEMLIMIT` is restored after spawning. +This is a Go garbage-collector soft limit, not a hard combined RSS ceiling: +Node allocations during extraction, native non-Go allocations, and oversized +individual files can exceed the budget. Closing the native process after every +file prevents native caches and heap capacity from accumulating across files. +Node RSS exceeding the budget produces a warning on stderr. + +Native stderr is inherited separately from the JSON stdout protocol. Syntax +diagnostics remain successful AST responses; native, I/O, and protocol failures +are reported on stderr and terminate the wrapper with a nonzero exit status. +Reset, quit, EOF, and failure invalidate queued prefetch work; quit also closes +stdin without requiring the sender to close its pipe. + +TypeScript 7.1 parser spike +-------------------------- + +The `ts7` directory contains an isolated parser compatibility probe using the +published npm `typescript@next` package. The snapshot is pinned to +`7.1.0-dev.20261015.1` in `ts7/package.json`, with package integrity recorded in +`ts7/package-lock.json`. + +From this directory, run: + +```sh +cd ts7 +npm ci --ignore-scripts +npm test +``` + +Only Node.js and npm are needed; no source build or Go toolchain is required. +The probe has its own installation, including TypeScript 5.9.2 for comparison. +TypeScript's optional dependencies supply the host's matching native executable. +To update the snapshot, verify that `next` still targets 7.1 and run +`npm install --save-exact typescript@next` in `ts7`, then rerun the probe and +review the manifest and lockfile. The probe has only been run on macOS arm64; +distribution packaging and its smoke test are described above. + +If a minimum-release-age policy blocks the nightly, obtain approval for a +command-local exception for `typescript` and `@typescript/*` rather than changing +saved npm configuration. + +The probe compares TS/TSX AST traversal and UTF-16 source positions, syntax-error +messages and offsets, and scanner output with 5.9. It also exercises standalone +source-file leases, program/snapshot disposal, repeated parsing, and API shutdown. +For syntax diagnostics, use a single-file program with `noResolve` and `noLib`; +standalone source-file leases do not expose the old `parseDiagnostics` field. +Compiler-option enums such as `JsxEmit.Preserve` must be passed as numbers, not +tsconfig-style strings. Diagnostic `text` and `pos` must be mapped to the old +`messageText` and `start` fields. AST properties must be explicitly projected +before JSON serialization because the new nodes expose them through getters. + +Wrapper source port +------------------- + +The wrapper targets the native `typescript/sync` and `typescript/ast` entry +points, using the same pinned dependency in the production ESM build. +The probe command above also runs +`wrapper.test.mjs`, which transpiles the wrapper into an isolated ESM directory +using the probe's 5.9 compiler and exercises the actual JSON protocol with the +pinned 7.1 runtime. + +AST nodes are explicitly projected into plain objects before their native +program is disposed. The projection preserves the Java converter's property +names, including optional member tokens, type-parameter defaults, import +attributes, and namespace flags. EOF metadata retains the `EndOfFileToken` +spelling; the native `TypeOfExpression`, `TypeOfKeyword`, `InstanceOfKeyword`, +and `KeyOfKeyword` names already match the Java converter. + +`npm test` in the wrapper directory instead exercises the compiled production +output, copied with its ESM package boundary into a temporary installation. It +covers repeated parses, native process disposal, the child memory budget, +reset/quit/EOF, cancelled prefetch, runtime overrides, and fatal native failures. + +Compatibility and regression coverage +------------------------------------ + +Native interface `extends` and class `implements` elements are type references. +The wrapper projects them into the expression-shaped heritage nodes consumed by +the Java converter, preserving generic arguments and qualified names. + +The native parser still produces ASTs for legacy `assert { ... }` import +assertions, but adds diagnostic 2880. The wrapper excludes only that diagnostic +to preserve previously accepted syntax; genuine syntax diagnostics are unchanged. +Import-attribute keys and values are now explicitly serialized. Their absence +from the old wrapper's JSON was a bug: the Java converter expects `key`/`value`, +but the old filter exposed neither correctly. The corresponding TRAP expectation +was regenerated; existing QL expectations did not need changes. + +Rescanning preserves token text and offsets while retaining the original +pre-rescan token kinds on the wire. In particular, regex and template-tail tokens +keep the classifications expected by existing TRAP tests. + +Source-phase imports (`import source wasm from "./module.wasm"`) are accepted. +Their declaration, imported path, and variable bindings use the existing import +representation; the schema does not yet distinguish the source phase. The new +`TypeScript/SourceImports` QL test covers these bindings. + +Validation on October 19, 2026 passed the pinned-compiler build/type check, +existing TSLint rules, seven compiled-output tests, 23 probe/compatibility tests, +500 Java tests, 68 TypeScript QL tests, the syntax-error query test, both extraction +diagnostic integration tests, and the installed-package smoke test. The +internal-error integration test deliberately terminates the native child rather +than relying on the old parser's stack overflow. + +For QL tests, install the newly built extractor and explicitly select it; otherwise +the CLI may silently exercise an older installed extractor: + +```sh +# From the repository root, with the Java extractor build dependencies available: +bazel run //javascript:install +codeql resolve languages --search-path=javascript/extractor-pack:. --format=json +codeql test run -j4 --search-path=javascript/extractor-pack:. \ + javascript/ql/test/library-tests/TypeScript \ + javascript/ql/test/query-tests/LanguageFeatures/SyntaxError +``` + +Performance comparison +---------------------- + +On macOS arm64 with Node.js 18.15.0, a comparison against the pre-migration +wrapper from `1971c368e1d` compiled with TypeScript 5.9.2 gave these medians over three +runs: + +| Workload | 5.9.2 time | 7.1 time | 5.9.2 combined RSS | 7.1 combined RSS | +| --- | --- | --- | --- | --- | +| 103 TypeScript fixtures (33,589 bytes) | 0.237 s | 1.349 s | 124.1 MiB | 113.9 MiB | +| Five parses of TypeScript's 588,128-byte declaration file | 0.991 s | 1.116 s | 269.3 MiB | 249.1 MiB | + +Timing covers prepare-files, extraction, serialization, and protocol responses +after the metadata handshake, not Java conversion or database import. Combined +RSS is the maximum sampled sum of Node and its native descendants, sampled every +5 ms; short-lived peaks may be missed. The per-file native process costs about +5.7x on tiny files and 1.13x on the declaration workload, while measured memory is +slightly lower. The bounded lifecycle is retained deliberately; this migration +does not claim a speedup. diff --git a/javascript/extractor/lib/typescript/package-lock.json b/javascript/extractor/lib/typescript/package-lock.json index 68923719ff8d..dd7822deee01 100644 --- a/javascript/extractor/lib/typescript/package-lock.json +++ b/javascript/extractor/lib/typescript/package-lock.json @@ -6,10 +6,13 @@ "": { "name": "typescript-parser-wrapper", "dependencies": { - "typescript": "5.9" + "typescript": "7.1.0-dev.20261015.1" }, "devDependencies": { "@types/node": "^24.3.1" + }, + "engines": { + "node": ">=18.15.0" } }, "node_modules/@types/node": { @@ -21,16 +24,137 @@ "undici-types": "~7.10.0" } }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-cIBrgsPrcPJfnyEP1gnonNoBcV6WZAxpWdrOO6F60eDhwknkHPxcTmdwuFA9ar07fFzyfp7XhKRrqz7OlrSQnA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-TkxvjvYuMDSzEmeI3g6XzLPKb6FBlonWu9FkmLMeXMNAxWRBjJmXdyKEBhDM3EASDBXAZL68X2sEim1l7e9r4g==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-8hnNa5ptGrsXCl5bs7ijxVQsuOcaCGxeDOAWqFdCNIRDQ9Z69NNrmizH650gPnZQCcNyxkFpK7QmoGSv9jsHSg==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-LaWnTy/FahvhOdo8w2ugYkDAmbcBuY6Zm8jNKYtqghx8bGeH0Mj+dX4533++tzPxAZoRnCE0WdGoBfzpDlLS8Q==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-9hRSH3i84yqjpVZtapYH/GAvRxTnh71Bui6EQDWkvrBZ6wvFleEgsyEg9RXxQDedXNMMHwKRGSYT7NvRymuytw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-hUdo1fnWfYsC4Az+1uBySKCaLBvrfWEH+jALFFvY5fROBU7cqoobVRQmOweJpiLsUQDqrHXAD3v0rnCEhCKI9w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-SQephnLxrS46fwPsM2K1LGbD8jHT4H4KF+bBoGnaSoEMLgLulAdrhY7A7lWUI3OMRuwawjfHxsnSV4TjeKEhcw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/typescript": { - "version": "5.9.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.2.tgz", - "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==", + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-IonOVC0CPC24brjf3BOPYSrU8CioR/p15jRRUnx6M6zywu4tcDw0vcIKE1dR9XpOEpArd15xZ5rIFj9wjAyOkg==", + "license": "Apache-2.0", "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" + "tsc": "bin/tsc" }, "engines": { - "node": ">=14.17" + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-darwin-arm64": "7.1.0-dev.20261015.1", + "@typescript/typescript-darwin-x64": "7.1.0-dev.20261015.1", + "@typescript/typescript-linux-arm": "7.1.0-dev.20261015.1", + "@typescript/typescript-linux-arm64": "7.1.0-dev.20261015.1", + "@typescript/typescript-linux-x64": "7.1.0-dev.20261015.1", + "@typescript/typescript-win32-arm64": "7.1.0-dev.20261015.1", + "@typescript/typescript-win32-x64": "7.1.0-dev.20261015.1" } }, "node_modules/undici-types": { diff --git a/javascript/extractor/lib/typescript/package-runtime.mjs b/javascript/extractor/lib/typescript/package-runtime.mjs new file mode 100644 index 000000000000..b235994784f5 --- /dev/null +++ b/javascript/extractor/lib/typescript/package-runtime.mjs @@ -0,0 +1,39 @@ +import { chmodSync, mkdirSync, readFileSync, renameSync, rmSync, statSync } from "node:fs"; +import { join } from "node:path"; + +const platforms = { + "linux64": ["linux-x64"], + "linux-arm64": ["linux-arm64"], + "osx64": ["darwin-arm64", "darwin-x64"], + "win64": ["win32-x64"], +}; +const packagesOnly = process.argv[2] === "--packages"; +const distribution = process.argv[packagesOnly ? 3 : 2]; +if (!Object.hasOwn(platforms, distribution)) { + throw new Error("Unsupported CodeQL distribution: " + distribution); +} +const readJson = filename => JSON.parse(readFileSync(filename, "utf8")); +const typescript = readJson("node_modules/typescript/package.json"); +if (packagesOnly) { + console.log(platforms[distribution] + .map(platform => "@typescript/typescript-" + platform + "@" + typescript.version).join(" ")); + process.exit(0); +} +mkdirSync("native"); +for (const platform of platforms[distribution]) { + const name = "typescript-" + platform; + const source = join("node_modules", "@typescript", name); + const manifest = readJson(join(source, "package.json")); + if (manifest.name !== "@typescript/" + name + || manifest.version !== typescript.version + || typescript.optionalDependencies[manifest.name] !== manifest.version) { + throw new Error("Native runtime does not match TypeScript: " + manifest.name); + } + const executable = join(source, "lib", platform.startsWith("win32-") ? "tsc.exe" : "tsc"); + if (!statSync(executable).isFile()) { + throw new Error("Native executable is not a file: " + executable); + } + chmodSync(executable, 0o755); + renameSync(source, join("native", name)); +} +rmSync("node_modules/@typescript", { recursive: true }); diff --git a/javascript/extractor/lib/typescript/package.json b/javascript/extractor/lib/typescript/package.json index f23f197e870b..732cb4f0c49b 100644 --- a/javascript/extractor/lib/typescript/package.json +++ b/javascript/extractor/lib/typescript/package.json @@ -1,15 +1,20 @@ { "name": "typescript-parser-wrapper", "private": true, + "type": "module", + "engines": { + "node": ">=18.15.0" + }, "dependencies": { - "typescript": "5.9" + "typescript": "7.1.0-dev.20261015.1" }, "scripts": { - "build": "tsc --project tsconfig.json", - "check": "tsc --noEmit --project . && tslint --project .", + "build": "node node_modules/typescript/lib/tsc.js --project tsconfig.json", + "check": "node node_modules/typescript/lib/tsc.js --noEmit --project . && tslint --project .", + "test": "npm run build && node --test test/lifecycle.test.mjs", "lint": "tslint --project .", "lint-fix": "tslint --project . --fix", - "watch": "tsc -p . -w --sourceMap" + "watch": "node node_modules/typescript/lib/tsc.js -p . -w --sourceMap" }, "devDependencies": { "@types/node": "^24.3.1" diff --git a/javascript/extractor/lib/typescript/src/ast_extractor.ts b/javascript/extractor/lib/typescript/src/ast_extractor.ts index 3dbc8a52d641..bfa49ea8acfd 100644 --- a/javascript/extractor/lib/typescript/src/ast_extractor.ts +++ b/javascript/extractor/lib/typescript/src/ast_extractor.ts @@ -1,177 +1,227 @@ -import * as ts from "./typescript"; -import { Project } from "./common"; +import * as ts from "./typescript.js"; -/** - * Interface that exposes some internal properties we rely on, as well as - * some properties we add ourselves. - */ -export interface AugmentedSourceFile extends ts.SourceFile { - parseDiagnostics?: any[]; - /** Internal property that we expose as a workaround. */ - redirectInfo?: object | null; - $tokens?: Token[]; - $lineStarts?: ReadonlyArray; +export type JsonValue = string | number | boolean | JsonObject | JsonValue[]; +export interface JsonObject { + [property: string]: JsonValue; } -export interface AugmentedNode extends ts.Node { - $pos?: any; - $end?: any; - $declarationKind?: string; -} - -export type AugmentedPos = number; - -export interface Token { - kind: ts.SyntaxKind; - tokenPos: AugmentedPos; - text: string; -} - -function hasOwnProperty(o: object, p: string) { - return o && Object.prototype.hasOwnProperty.call(o, p); -} - -// build our own copy of `ts.SyntaxKind` without `First*`/`Last*` markers -let SyntaxKind: { [id: number]: string } = []; -for (let p in ts.SyntaxKind) { - if (!hasOwnProperty(ts.SyntaxKind, p)) { - continue; - } - // skip numeric indices - if (+p === +p) { - continue; - } - // skip `First*`/`Last*` - if (p.substring(0, 5) === "First" || p.substring(0, 4) === "Last") { - continue; +/** Child and scalar properties consumed by the Java AST converter. */ +const astProperties = [ + "argument", + "argumentExpression", + "arguments", + "assertsModifier", + "asteriskToken", + "attributes", + "block", + "body", + "caseBlock", + "catchClause", + "checkType", + "children", + "clauses", + "closingElement", + "closingFragment", + "condition", + "constraint", + "declarationList", + "declarations", + "dotDotDotToken", + "elements", + "elementType", + "elseStatement", + "exclamationToken", + "exportClause", + "expression", + "exprName", + "extendsType", + "falseType", + "finallyBlock", + "head", + "heritageClauses", + "importClause", + "incrementor", + "indexType", + "initializer", + "keywordToken", + "label", + "left", + "literal", + "members", + "modifiers", + "moduleReference", + "moduleSpecifier", + "name", + "namedBindings", + "objectType", + "openingElement", + "openingFragment", + "operand", + "operator", + "operatorToken", + "parameterName", + "parameters", + "phaseModifier", + "properties", + "propertyName", + "qualifier", + "questionDotToken", + "questionToken", + "right", + "statement", + "statements", + "tag", + "tagName", + "template", + "templateSpans", + "text", + "thenStatement", + "token", + "trueType", + "tryBlock", + "type", + "typeArguments", + "typeName", + "typeParameter", + "typeParameters", + "types", + "variableDeclaration", + "whenFalse", + "whenTrue", +]; + +const skipWhiteSpace = /(?:\s|\/\/.*|\/\*[^]*?\*\/)*/g; + +export function extractAst(ast: ts.SourceFile, diagnostics: readonly ts.Diagnostic[]): JsonObject { + const code = ast.text; + // The native parser still builds assertion ASTs, but now diagnoses this formerly accepted syntax. + const parseDiagnostics = diagnostics.filter(diagnostic => diagnostic.code !== 2880); + + function augmentPos(pos: number): number { + skipWhiteSpace.lastIndex = pos; + return pos + skipWhiteSpace.exec(code)![0].length; } - SyntaxKind[ts.SyntaxKind[p] as any] = p; -} - -let skipWhiteSpace = /(?:\s|\/\/.*|\/\*[^]*?\*\/)*/g; -/** - * Invokes the given callback for every AST node in the given tree. - */ -function forEachNode(ast: ts.Node, callback: (node: ts.Node) => void) { - function visit(node: ts.Node) { - ts.forEachChild(node, visit); - callback(node); - } - visit(ast); -} - -function tryGetTypeOfNode(typeChecker: ts.TypeChecker, node: AugmentedNode): ts.Type | null { - try { - return typeChecker.getTypeAtLocation(node); - } catch (e) { - let sourceFile = node.getSourceFile(); - let { line, character } = sourceFile.getLineAndCharacterOfPosition(node.pos); - console.warn(`Could not compute type of ${ts.SyntaxKind[node.kind]} at ${sourceFile.fileName}:${line + 1}:${character + 1}`); - return null; + function projectValue(value: unknown): JsonValue { + if (Array.isArray(value)) return Array.from(value, projectValue); + if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { + return value; + } + if (value != null && typeof value === "object" && "forEachChild" in value) { + return projectNode(value as ts.Node); + } + throw new Error("Unexpected TypeScript AST property value"); } -} -export function augmentAst(ast: AugmentedSourceFile, code: string, project: Project | null) { - ast.$lineStarts = ast.getLineStarts(); + function projectNode(node: ts.Node): JsonObject { + const result: JsonObject = { + kind: node.kind, + flags: node.flags, + $pos: augmentPos(node.pos), + $end: node.end, + }; + const properties = node as ts.Node & { [property: string]: unknown }; + for (const property of astProperties) { + const value = properties[property]; + if (value != null) result[property] = projectValue(value); + } - /** - * Converts a numeric offset to the value expected by the Java counterpart of the extractor. - */ - function augmentPos(pos: number, shouldSkipWhitespace?: boolean): AugmentedPos { - // skip over leading spaces/comments - if (shouldSkipWhitespace) { - skipWhiteSpace.lastIndex = pos; - pos += skipWhiteSpace.exec(code)[0].length; + // Native nodes share boolean getters even on kinds where they are not meaningful. + if (ts.isExportAssignment(node)) result.isExportEquals = node.isExportEquals; + if (ts.isExportDeclaration(node) || ts.isImportEqualsDeclaration(node) + || ts.isImportSpecifier(node) || ts.isExportSpecifier(node)) { + result.isTypeOnly = node.isTypeOnly; } - return pos; - } + if (ts.isImportTypeNode(node)) result.isTypeOf = node.isTypeOf; - // Find the position of all tokens where the scanner requires parse-tree information. - // At these offsets, a call to `scanner.reScanX` is required. - type RescanEvent = () => void; - let reScanEvents: RescanEvent[] = []; - let reScanEventPos: number[] = []; - let scanner = ts.createScanner(ts.ScriptTarget.ES2015, false, 1, code); - let reScanSlashToken = scanner.reScanSlashToken.bind(scanner); - let reScanTemplateToken = scanner.reScanTemplateToken.bind(scanner); - let reScanGreaterToken = scanner.reScanGreaterToken.bind(scanner); - if (!ast.parseDiagnostics || ast.parseDiagnostics.length === 0) { - forEachNode(ast, node => { - if (ts.isRegularExpressionLiteral(node)) { - reScanEventPos.push(node.getStart(ast, false)); - reScanEvents.push(reScanSlashToken); + if ("postfixToken" in node) { + const token = (node as ts.NamedMemberBase).postfixToken; + if (token != null) { + const property = token.kind === ts.SyntaxKind.QuestionToken ? "questionToken" : "exclamationToken"; + result[property] = projectNode(token); } - if (ts.isTemplateMiddle(node) || ts.isTemplateTail(node)) { - reScanEventPos.push(node.getStart(ast, false)); - reScanEvents.push(reScanTemplateToken); + } + if (ts.isTypeParameterDeclaration(node) && node.defaultType != null) { + result.default = projectNode(node.defaultType); + } + if (ts.isTypeReferenceNode(node) && ts.isHeritageClause(node.parent)) { + result.kind = ts.SyntaxKind.ExpressionWithTypeArguments; + result.expression = projectNode(node.typeName); + delete result.typeName; + } + if (ts.isImportAttributes(node)) result.elements = node.attributes.map(projectNode); + if (ts.isImportAttribute(node)) { + result.key = projectNode(node.name); + result.value = projectNode(node.value); + } + if (ts.isVariableDeclarationList(node)) { + result.$declarationKind = node.flags & ts.NodeFlags.Let ? "let" + : node.flags & ts.NodeFlags.Const ? "const" : "var"; + } + if (ts.isModuleDeclaration(node)) { + let flags = node.flags; + if (ts.isModuleDeclaration(node.parent) && node.parent.body === node) { + flags |= ts.nodeFlags.NestedNamespace; } - if (ts.isBinaryExpression(node)) { - let operator = node.operatorToken; - switch (operator.kind) { - case ts.SyntaxKind.GreaterThanEqualsToken: - case ts.SyntaxKind.GreaterThanGreaterThanEqualsToken: - case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken: - case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanToken: - case ts.SyntaxKind.GreaterThanGreaterThanToken: - reScanEventPos.push(operator.getStart(ast, false)); - reScanEvents.push(reScanGreaterToken); - break; - } + if (ts.isIdentifier(node.name) && node.name.text === "global" + && ts.skipTrivia(code, node.modifiers == null ? node.pos : node.modifiers.end) + === node.name.getStart(ast)) { + flags |= ts.nodeFlags.GlobalAugmentation; } - }); - } - - reScanEventPos.push(Infinity); - - // add tokens and comments to the AST - ast.$tokens = []; - let rescanEventIndex = 0; - let nextRescanPosition = reScanEventPos[0]; - let tk; - do { - tk = scanner.scan(); - if (scanner.getTokenPos() === nextRescanPosition) { - let callback = reScanEvents[rescanEventIndex]; - callback(); - ++rescanEventIndex; - nextRescanPosition = reScanEventPos[rescanEventIndex]; + result.flags = flags; } - ast.$tokens.push({ - kind: tk, - tokenPos: augmentPos(scanner.getTokenPos()), - text: scanner.getTokenText(), - }); - } while (tk !== ts.SyntaxKind.EndOfFileToken); - - if (ast.parseDiagnostics) { - ast.parseDiagnostics.forEach(d => { - delete d.file; - d.$pos = augmentPos(d.start); - }); + return result; } - visitAstNode(ast); - function visitAstNode(node: AugmentedNode) { - ts.forEachChild(node, visitAstNode); - - // fill in line/column info - if ("pos" in node) { - node.$pos = augmentPos(node.pos, true); - } - if ("end" in node) { - node.$end = augmentPos(node.end); - } + const result = projectNode(ast); + result.$lineStarts = Array.from(ast.getLineStarts()); + result.$tokens = extractTokens(ast, parseDiagnostics.length === 0); + result.parseDiagnostics = parseDiagnostics.map(diagnostic => ({ + messageText: diagnostic.text, + $pos: diagnostic.pos, + })); + return result; +} - if (ts.isVariableDeclarationList(node as any)) { - let tz = ts as any; - if (typeof tz.isLet === "function" && tz.isLet(node) || (node.flags & ts.NodeFlags.Let)) { - node.$declarationKind = "let"; - } else if (typeof tz.isConst === "function" && tz.isConst(node) || (node.flags & ts.NodeFlags.Const)) { - node.$declarationKind = "const"; - } else { - node.$declarationKind = "var"; +function extractTokens(ast: ts.SourceFile, rescan: boolean): JsonObject[] { + const scanner = ts.createScanner(false, ts.LanguageVariant.JSX, ast.text); + const rescans = new Map ts.SyntaxKind>(); + if (rescan) visit(ast); + + function visit(node: ts.Node): void { + node.forEachChild(visit); + if (ts.isRegularExpressionLiteral(node)) { + rescans.set(node.getStart(ast), () => scanner.reScanSlashToken()); + } else if (ts.isTemplateMiddle(node) || ts.isTemplateTail(node)) { + const template = node.parent.parent; + const tagged = ts.isTemplateExpression(template) && ts.isTaggedTemplateExpression(template.parent); + rescans.set(node.getStart(ast), () => scanner.reScanTemplateToken(tagged)); + } else if (ts.isBinaryExpression(node)) { + const operator = node.operatorToken; + switch (operator.kind) { + case ts.SyntaxKind.GreaterThanEqualsToken: + case ts.SyntaxKind.GreaterThanGreaterThanEqualsToken: + case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken: + case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanToken: + case ts.SyntaxKind.GreaterThanGreaterThanToken: + rescans.set(operator.getStart(ast), () => scanner.reScanGreaterToken()); + break; } } } + + const tokens: JsonObject[] = []; + let kind: ts.SyntaxKind; + do { + kind = scanner.scan(); + const callback = rescans.get(scanner.getTokenStart()); + // Keep the pre-rescan kind, as in the original Java-facing token protocol. + if (callback != null) callback(); + tokens.push({ + kind, + tokenPos: scanner.getTokenStart(), + text: scanner.getTokenText(), + }); + } while (kind !== ts.SyntaxKind.EndOfFile); + return tokens; } diff --git a/javascript/extractor/lib/typescript/src/common.ts b/javascript/extractor/lib/typescript/src/common.ts deleted file mode 100644 index 9276a567bb08..000000000000 --- a/javascript/extractor/lib/typescript/src/common.ts +++ /dev/null @@ -1,35 +0,0 @@ -import * as ts from "./typescript"; - -export class Project { - public program: ts.Program = null; - private host: ts.CompilerHost; - - constructor( - public tsConfig: string, - public config: ts.ParsedCommandLine, - public packageEntryPoints: Map) { - - let host = ts.createCompilerHost(config.options, true); - host.trace = undefined; // Disable tracing which would otherwise go to standard out - this.host = host; - } - - public unload(): void { - this.program = null; - } - - public load(): void { - const { config, host } = this; - this.program = ts.createProgram(config.fileNames, config.options, host); - } - - /** - * Discards the old compiler instance and starts a new one. - */ - public reload(): void { - // Ensure all references to the old compiler instance - // are cleared before calling `createProgram`. - this.unload(); - this.load(); - } -} diff --git a/javascript/extractor/lib/typescript/src/main.ts b/javascript/extractor/lib/typescript/src/main.ts index 0630213c7431..447496bfc959 100644 --- a/javascript/extractor/lib/typescript/src/main.ts +++ b/javascript/extractor/lib/typescript/src/main.ts @@ -1,58 +1,19 @@ /** - * @fileoverview Wrapper for TypeScript parser, inspired by - * [typescript-eslint-parser](https://github.com/eslint/typescript-eslint-parser). - * - * The wrapper can be invoked with a single argument, which is interpreted as a file name. - * The file is parsed and the AST dumped to stdout as described below. - * - * When run without arguments, the wrapper enters server mode, expecting JSON-encoded requests - * on `stdin`, one per line. - * - * Each request should have a `command` property, which should either be `parse` or `quit`. - * - * A `quit` request causes the wrapper to terminate. - * - * A `parse` request should have a `filename` property, which should be the path of a - * TypeScript file. - * - * In response to a `parse` request, the wrapper invokes the TypeScript parser to parse - * the given file, and dumps a JSON representation of the resulting AST to stdout, - * with the following adjustments: - * - * - the `kind` and `operator` properties have symbolic string values instead of numeric - * values (the latter change between versions of the TypeScript compiler, the former - * don't); - * - position information is given in terms of line/column pairs instead of offsets; - * - declarations have a `declarationKind` property indicating whether they are `var`, - * `let` or `const` declarations; - * - `parent` pointers are omitted (since they cannot be serialised to JSON); - * - tokens and comments are added to the AST root node in a `tokens` property. + * Wrapper for the TypeScript parser. Accepts newline-delimited JSON commands on stdin + * and writes ASTs and parser metadata to stdout, or parses a single command-line file. */ -"use strict"; - import * as pathlib from "path"; import * as readline from "readline"; -import * as ts from "./typescript"; -import * as ast_extractor from "./ast_extractor"; - -import { Project } from "./common"; -import { VirtualSourceRoot } from "./virtual_source_root"; +import * as ts from "./typescript.js"; +import { extractAst } from "./ast_extractor.js"; -// Remove limit on stack trace depth. Error.stackTraceLimit = Infinity; interface ParseCommand { command: "parse"; filename: string; } -interface LoadCommand { - tsConfig: string; - sourceRoot: string | null; - virtualSourceRoot: string | null; - packageEntryPoints: [string, string][]; - packageJsonFiles: [string, string][]; -} interface ResetCommand { command: "reset"; } @@ -68,543 +29,170 @@ interface GetMetadataCommand { } type Command = ParseCommand | ResetCommand | QuitCommand | PrepareFilesCommand | GetMetadataCommand; -/** The state to be shared between commands. */ class State { - public project: Project = null; - - /** List of files that have been requested. */ public pendingFiles: string[] = []; public pendingFileIndex = 0; - - /** Next response to be delivered. */ - public pendingResponse: string = null; - - /** Map from `package.json` files to their contents. */ - public parsedPackageJson = new Map(); - - /** Map from `package.json` files to the file referenced in its `types` or `typings` field. */ - public packageTypings = new Map(); - - /** Map from file path to the enclosing `package.json` file, if any. Will not traverse outside node_modules. */ - public enclosingPackageJson = new Map(); + public pendingResponse: string | null = null; } let state = new State(); - -const reloadMemoryThresholdMb = getEnvironmentVariable("SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD", Number, 1000); - -function getPackageJson(file: string): any { - let cache = state.parsedPackageJson; - if (cache.has(file)) return cache.get(file); - let result = getPackageJsonRaw(file); - cache.set(file, result); - return result; -} - -function getPackageJsonRaw(file: string): any { - if (!ts.sys.fileExists(file)) return undefined; - try { - let json = JSON.parse(ts.sys.readFile(file)); - if (typeof json !== 'object') return undefined; - return json; - } catch (e) { - return undefined; +let stopped = false; +let readLineInterface: readline.Interface | undefined; +let reportedMemoryPressure = false; + +function createAPI(): ts.API { + const threshold = Number(process.env.SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD ?? 1000) * 1000000; + if (!Number.isSafeInteger(threshold) || threshold <= 0) { + throw new Error("SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD must be a positive number of megabytes"); } -} - -function getPackageTypings(file: string): string | undefined { - let cache = state.packageTypings; - if (cache.has(file)) return cache.get(file); - let result = getPackageTypingsRaw(file); - cache.set(file, result); - return result; -} - -function getPackageTypingsRaw(packageJsonFile: string): string | undefined { - let json = getPackageJson(packageJsonFile); - if (json == null) return undefined; - let typings = json.types || json.typings; // "types" and "typings" are aliases - if (typeof typings !== 'string') return undefined; - let absolutePath = pathlib.join(pathlib.dirname(packageJsonFile), typings); - if (ts.sys.directoryExists(absolutePath)) { - absolutePath = pathlib.join(absolutePath, 'index.d.ts'); - } else if (!absolutePath.endsWith('.ts')) { - absolutePath += '.d.ts'; + const rss = process.memoryUsage().rss; + if (rss >= threshold && !reportedMemoryPressure) { + console.warn("Node.js RSS exceeds the TypeScript memory budget; limiting native compiler memory"); } - if (!ts.sys.fileExists(absolutePath)) return undefined; - return ts.sys.resolvePath(absolutePath); -} + reportedMemoryPressure = rss >= threshold; -function getEnclosingPackageJson(file: string): string | undefined { - let cache = state.packageTypings; - if (cache.has(file)) return cache.get(file); - let result = getEnclosingPackageJsonRaw(file); - cache.set(file, result); - return result; -} - -function getEnclosingPackageJsonRaw(file: string): string | undefined { - let packageJson = pathlib.join(file, 'package.json'); - if (ts.sys.fileExists(packageJson)) { - return packageJson; - } - if (pathlib.basename(file) === 'node_modules') { - return undefined; - } - let dirname = pathlib.dirname(file); - if (dirname.length < file.length) { - return getEnclosingPackageJson(dirname); + // The API inherits the environment when it spawns its native process. + const previousLimit = process.env.GOMEMLIMIT; + process.env.GOMEMLIMIT = String(Math.max(1, threshold - rss)); + try { + return new ts.API(); + } finally { + if (previousLimit == null) delete process.env.GOMEMLIMIT; + else process.env.GOMEMLIMIT = previousLimit; } - return undefined; } -/** - * Debugging method for finding cycles in the TypeScript AST. Should not be used in production. - * - * If cycles are found, the whitelist in `astProperties` is too permissive. - */ -// tslint:disable-next-line:no-unused-variable -function checkCycle(root: any) { - let path: string[] = []; - function visit(obj: any): boolean { - if (obj == null || typeof obj !== "object") return false; - if (obj.$cycle_visiting) { - return true; - } - obj.$cycle_visiting = true; - for (let k in obj) { - if (!obj.hasOwnProperty(k)) continue; - // Ignore numeric and whitelisted properties. - if (+k !== +k && !astPropertySet.has(k)) continue; - if (k === "$cycle_visiting") continue; - let cycle = visit(obj[k]); - if (cycle) { - path.push(k); - return true; +function extractFile(filename: string): string { + const api = createAPI(); + try { + const program = api.createProgram([pathlib.resolve(filename)], { + experimentalDecorators: true, + jsx: ts.JsxEmit.Preserve, + noResolve: true, + noLib: true, + }); + try { + const ast = program.getSourceFile(pathlib.resolve(filename)); + if (ast == null) { + throw new Error("TypeScript did not return a source file for " + filename); } + return JSON.stringify({ + type: "ast", + ast: extractAst(ast, program.getSyntacticDiagnostics()), + }); + } finally { + program.dispose(); } - obj.$cycle_visiting = undefined; - return false; + } finally { + api.close(); } - visit(root); - if (path.length > 0) { - path.reverse(); - console.log(JSON.stringify({ type: "error", message: "Cycle = " + path.join(".") })); - } -} - -/** Property names to extract from the TypeScript AST. */ -const astProperties: string[] = [ - "$declarationKind", - "$end", - "$lineStarts", - "$overloadIndex", - "$pos", - "$tokens", - "argument", - "argumentExpression", - "arguments", - "assertsModifier", - "asteriskToken", - "attributes", - "block", - "body", - "caseBlock", - "catchClause", - "checkType", - "children", - "clauses", - "closingElement", - "closingFragment", - "condition", - "constraint", - "constructor", - "declarationList", - "declarations", - "default", - "delete", - "dotDotDotToken", - "elements", - "elementType", - "elementTypes", - "elseStatement", - "escapedText", - "exclamationToken", - "exportClause", - "expression", - "exprName", - "extendsType", - "falseType", - "finallyBlock", - "flags", - "head", - "heritageClauses", - "importClause", - "incrementor", - "indexType", - "init", - "initializer", - "isExportEquals", - "isTypeOf", - "isTypeOnly", - "keywordToken", - "kind", - "label", - "left", - "literal", - "members", - "messageText", - "modifiers", - "moduleReference", - "moduleSpecifier", - "name", - "namedBindings", - "objectType", - "openingElement", - "openingFragment", - "operand", - "operator", - "operatorToken", - "parameterName", - "parameters", - "parseDiagnostics", - "phaseModifier", - "properties", - "propertyName", - "qualifier", - "questionDotToken", - "questionToken", - "right", - "selfClosing", - "statement", - "statements", - "tag", - "tagName", - "template", - "templateSpans", - "text", - "thenStatement", - "token", - "tokenPos", - "trueType", - "tryBlock", - "type", - "typeArguments", - "typeName", - "typeParameter", - "typeParameters", - "types", - "variableDeclaration", - "whenFalse", - "whenTrue", -]; - -/** Property names used in a parse command response, in addition to the AST itself. */ -const astMetaProperties: string[] = [ - "ast", - "type", -]; - -/** Property names to extract in an AST response. */ -const astPropertySet = new Set([...astProperties, ...astMetaProperties]); - -/** - * Converts (part of) an AST to a JSON string, ignoring properties we're not interested in. - */ -function stringifyAST(obj: any) { - return JSON.stringify(obj, (k, v) => { - // Filter out properties that aren't numeric, empty, or whitelisted. - // Note `k` is the empty string for the root object, which is also covered by +k === +k. - return (+k === +k || astPropertySet.has(k)) ? v : undefined; - }); -} - -function extractFile(filename: string): string { - let ast = getAstForFile(filename); - - return stringifyAST({ - type: "ast", - ast, - }); } function prepareNextFile() { if (state.pendingResponse != null) return; if (state.pendingFileIndex < state.pendingFiles.length) { - checkMemoryUsage(); - let nextFilename = state.pendingFiles[state.pendingFileIndex]; - state.pendingResponse = extractFile(nextFilename); + state.pendingResponse = extractFile(state.pendingFiles[state.pendingFileIndex]); } } -function handleParseCommand(command: ParseCommand, checkPending = true) { - let filename = command.filename; - let expectedFilename = state.pendingFiles[state.pendingFileIndex]; - if (expectedFilename !== filename && checkPending) { - // File was requested out of order. This happens in rare cases because the Java process decided against extracting it, - // for example because it was too large. Just recover and accept that some work was wasted. - state.pendingResponse = null; - state.pendingFileIndex = state.pendingFiles.indexOf(filename); - } - ++state.pendingFileIndex; - let response = state.pendingResponse || extractFile(command.filename); - state.pendingResponse = null; - process.stdout.write(response + "\n", () => { - // Start working on the next file as soon as the old one is flushed. - // Note that if we didn't wait for flushing, this would block the I/O - // loop and delay flushing. - prepareNextFile(); - }); -} - -/** - * Returns true if the given source file has an actual AST, as opposed to - * a SourceFile that is a "redirect" to another SourceFile. - * - * The TypeScript API should not expose such redirecting SourceFiles, - * but we sometimes get them anyway. - */ -function isExtractableSourceFile(ast: ast_extractor.AugmentedSourceFile): boolean { - return ast.redirectInfo == null; -} - -/** - * Gets the AST and source code for the given file, either from - * an already-open project, or by parsing the file. - */ -function getAstForFile(filename: string): ts.SourceFile { - let { ast, code } = parseSingleFile(filename); - if (ast != null && isExtractableSourceFile(ast)) { - ast_extractor.augmentAst(ast, code, null); - } - return ast; -} - -function parseSingleFile(filename: string): { ast: ts.SourceFile, code: string } { - let code = ts.sys.readFile(filename); - - // create a compiler host that only allows access to `filename` - let compilerHost: ts.CompilerHost = { - fileExists() { return true; }, - getCanonicalFileName() { return filename; }, - getCurrentDirectory() { return ""; }, - getDefaultLibFileName() { return "lib.d.ts"; }, - getNewLine() { return "\n"; }, - getSourceFile() { - return ts.createSourceFile(filename, code, ts.ScriptTarget.Latest, true); - }, - readFile() { return null; }, - useCaseSensitiveFileNames() { return true; }, - writeFile() { return null; }, - getDirectories() { return []; }, - }; - - // parse `filename` with minimial checking - let compilerOptions: ts.CompilerOptions = { - experimentalDecorators: true, - experimentalAsyncFunctions: true, - jsx: ts.JsxEmit.Preserve, - noResolve: true, - }; - - let program = ts.createProgram([filename], compilerOptions, compilerHost); - - let ast = program.getSourceFile(filename); - - return { ast, code }; -} - -/** - * Matches a path segment referencing a package in a node_modules folder, and extracts - * two capture groups: the package name, and the relative path in the package. - * - * For example `lib/node_modules/@foo/bar/src/index.js` extracts the capture groups [`@foo/bar`, `src/index.js`]. - */ -const nodeModulesRex = /[/\\]node_modules[/\\]((?:@[\w.-]+[/\\])?\w[\w.-]*)[/\\](.*)/; - -interface LoadedConfig { - config: ts.ParsedCommandLine; - basePath: string; - packageEntryPoints: Map; - packageJsonFiles: Map; - virtualSourceRoot: VirtualSourceRoot; - ownFiles: string[]; +function fail(error: unknown) { + stopped = true; + state = new State(); + readLineInterface?.close(); + process.stdin.destroy(); + console.error(error); + process.exitCode = 1; } -function loadTsConfig(command: LoadCommand): LoadedConfig { - let tsConfig = ts.readConfigFile(command.tsConfig, ts.sys.readFile); - let basePath = pathlib.dirname(command.tsConfig); - - let packageEntryPoints = new Map(command.packageEntryPoints); - let packageJsonFiles = new Map(command.packageJsonFiles); - let virtualSourceRoot = new VirtualSourceRoot(command.sourceRoot, command.virtualSourceRoot); - - /** - * Rewrites path segments of form `node_modules/PACK/suffix` to be relative to - * the location of package PACK in the source tree, if it exists. - */ - function redirectNodeModulesPath(path: string) { - let nodeModulesMatch = nodeModulesRex.exec(path); - if (nodeModulesMatch == null) return null; - let packageName = nodeModulesMatch[1]; - let packageJsonFile = packageJsonFiles.get(packageName); - if (packageJsonFile == null) return null; - let packageDir = pathlib.dirname(packageJsonFile); - let suffix = nodeModulesMatch[2]; - let finalPath = pathlib.join(packageDir, suffix); - if (!ts.sys.fileExists(finalPath)) return null; - return finalPath; - } - - /** - * Create the host passed to the tsconfig.json parser. - * - * We override its file system access in case there is an "extends" - * clause pointing into "./node_modules", which must be redirected to - * the location of an installed package or a checked-in package. - */ - let parseConfigHost: ts.ParseConfigHost = { - useCaseSensitiveFileNames: true, - readDirectory: (rootDir, extensions, excludes?, includes?, depth?) => { - // Perform the glob matching in both real and virtual source roots. - let exclusions = excludes == null ? [] : [...excludes]; - if (virtualSourceRoot.virtualSourceRoot != null) { - // qltest puts the virtual source root inside the real source root (.testproj). - // Make sure we don't find files inside the virtual source root in this pass. - exclusions.push(virtualSourceRoot.virtualSourceRoot); - } - let originalResults = ts.sys.readDirectory(rootDir, extensions, exclusions, includes, depth) - let virtualDir = virtualSourceRoot.toVirtualPath(rootDir); - if (virtualDir == null) { - return originalResults; - } - // Make sure glob matching does not to discover anything in node_modules. - let virtualExclusions = excludes == null ? [] : [...excludes]; - virtualExclusions.push('**/node_modules/**/*'); - let virtualResults = ts.sys.readDirectory(virtualDir, extensions, virtualExclusions, includes, depth) - return [...originalResults, ...virtualResults]; - }, - fileExists: (path: string) => { - return ts.sys.fileExists(path) - || virtualSourceRoot.toVirtualPathIfFileExists(path) != null - || redirectNodeModulesPath(path) != null; - }, - readFile: (path: string) => { - if (!ts.sys.fileExists(path)) { - let virtualPath = virtualSourceRoot.toVirtualPathIfFileExists(path); - if (virtualPath != null) return ts.sys.readFile(virtualPath); - virtualPath = redirectNodeModulesPath(path); - if (virtualPath != null) return ts.sys.readFile(virtualPath); - } - return ts.sys.readFile(path); +function afterResponse(expectedState: State) { + return (error?: Error | null) => { + if (error != null) { + fail(error); + return; + } + if (stopped || state !== expectedState) return; + try { + prepareNextFile(); + } catch (failure) { + fail(failure); } }; - let config = ts.parseJsonConfigFileContent(tsConfig.config, parseConfigHost, basePath); - - let ownFiles = config.fileNames.map(file => pathlib.resolve(file)); - - return { config, basePath, packageJsonFiles, packageEntryPoints, virtualSourceRoot, ownFiles }; } -function handleResetCommand(command: ResetCommand) { - reset(); - console.log(JSON.stringify({ - type: "reset-done", - })); -} - -function handlePrepareFilesCommand(command: PrepareFilesCommand) { - state.pendingFiles = command.filenames; - state.pendingFileIndex = 0; - state.pendingResponse = null; - process.stdout.write('{"type":"ok"}\n', () => { - prepareNextFile(); - }); -} - -function handleGetMetadataCommand(command: GetMetadataCommand) { - console.log(JSON.stringify({ - type: "metadata", - syntaxKinds: ts.SyntaxKind, - nodeFlags: ts.NodeFlags, - })); -} - -function reset() { - state = new State(); -} - -function getEnvironmentVariable(name: string, parse: (x: string) => T, defaultValue: T) { - let value = process.env[name]; - return value != null ? parse(value) : defaultValue; -} - -/** - * Whether the memory usage was last observed to be above the threshold for restarting the TypeScript compiler. - * - * This is to prevent repeatedly restarting the compiler if the GC does not immediately bring us below the - * threshold again. - */ -let hasReloadedSinceExceedingThreshold = false; - -/** - * If memory usage has moved above a the threshold, reboot the TypeScript compiler instance. - * - * Make sure to call this only when stdout has been flushed. - */ -function checkMemoryUsage() { - let bytesUsed = process.memoryUsage().heapUsed; - let megabytesUsed = bytesUsed / 1000000; - if (!hasReloadedSinceExceedingThreshold && megabytesUsed > reloadMemoryThresholdMb && state.project != null) { - console.warn('Restarting TypeScript compiler due to memory usage'); - state.project.reload(); - hasReloadedSinceExceedingThreshold = true; - } - else if (hasReloadedSinceExceedingThreshold && megabytesUsed < reloadMemoryThresholdMb) { - hasReloadedSinceExceedingThreshold = false; +function handleParseCommand(command: ParseCommand) { + const filename = command.filename; + const expectedFilename = state.pendingFiles[state.pendingFileIndex]; + if (expectedFilename !== filename) { + // Java may skip a prepared file, for example because it is too large. + state.pendingResponse = null; + state.pendingFileIndex = state.pendingFiles.indexOf(filename); } + ++state.pendingFileIndex; + const response = state.pendingResponse || extractFile(filename); + state.pendingResponse = null; + process.stdout.write(response + "\n", afterResponse(state)); } function runReadLineInterface() { - reset(); - let rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + readLineInterface = rl; + rl.on("close", () => { + stopped = true; + state = new State(); + process.stdin.destroy(); + }); + rl.on("error", fail); rl.on("line", (line: string) => { - let req: Command = JSON.parse(line); - switch (req.command) { - case "parse": - handleParseCommand(req); - break; - case "prepare-files": - handlePrepareFilesCommand(req); - break; - case "reset": - handleResetCommand(req); - break; - case "get-metadata": - handleGetMetadataCommand(req); - break; - case "quit": - rl.close(); - break; - default: - throw new Error("Unknown command " + (req as any).command + "."); + if (stopped) return; + try { + const req: Command = JSON.parse(line); + switch (req.command) { + case "parse": + handleParseCommand(req); + break; + case "prepare-files": + state = new State(); + state.pendingFiles = req.filenames; + process.stdout.write('{"type":"ok"}\n', afterResponse(state)); + break; + case "reset": + state = new State(); + console.log('{"type":"reset-done"}'); + break; + case "get-metadata": + console.log(JSON.stringify({ + type: "metadata", + syntaxKinds: ts.syntaxKinds, + nodeFlags: ts.nodeFlags, + })); + break; + case "quit": + rl.close(); + break; + default: + throw new Error("Unknown command " + (req as { command: string }).command + "."); + } + } catch (error) { + fail(error); } }); } -// Parse command-line arguments. -if (process.argv.length > 2) { - let argument = process.argv[2]; - if (argument === "--version") { - console.log("parser-wrapper with TypeScript " + ts.version); - } else if (pathlib.extname(argument) === ".ts" || pathlib.extname(argument) === ".tsx") { - handleParseCommand({ - command: "parse", - filename: argument, - }, false); +process.stdout.on("error", fail); + +try { + if (process.argv.length > 2) { + const argument = process.argv[2]; + if (argument === "--version") { + console.log("parser-wrapper with TypeScript " + ts.version); + } else if (pathlib.extname(argument) === ".ts" || pathlib.extname(argument) === ".tsx") { + process.stdout.write(extractFile(argument) + "\n"); + } else { + console.error("Unrecognized file or flag: " + argument); + process.exitCode = 1; + } } else { - console.error("Unrecognized file or flag: " + argument); + runReadLineInterface(); } - process.exit(0); -} else { - runReadLineInterface(); +} catch (error) { + fail(error); } diff --git a/javascript/extractor/lib/typescript/src/typescript.d.ts b/javascript/extractor/lib/typescript/src/typescript.d.ts index 769a318e7249..ebef2b4bfda9 100644 --- a/javascript/extractor/lib/typescript/src/typescript.d.ts +++ b/javascript/extractor/lib/typescript/src/typescript.d.ts @@ -1,2 +1,15 @@ -import * as ts from "typescript"; -export = ts; +export { + SyntaxKind, NodeFlags, LanguageVariant, createScanner, + isRegularExpressionLiteral, isTemplateMiddle, isTemplateTail, isTemplateExpression, + isTaggedTemplateExpression, isBinaryExpression, isVariableDeclarationList, isModuleDeclaration, + isIdentifier, isTypeParameterDeclaration, isImportAttributes, isImportAttribute, + isExportAssignment, isExportDeclaration, isImportEqualsDeclaration, isImportSpecifier, + isExportSpecifier, isImportTypeNode, isTypeReferenceNode, isHeritageClause, skipTrivia, +} from "typescript/ast"; +export type { Node, SourceFile, NamedMemberBase } from "typescript/ast"; +export { API, JsxEmit } from "typescript/sync"; +export type { Diagnostic } from "typescript/sync"; + +export const version: string; +export const syntaxKinds: { [name: string]: number }; +export const nodeFlags: typeof import("typescript/ast").NodeFlags & { GlobalAugmentation: number }; diff --git a/javascript/extractor/lib/typescript/src/typescript.js b/javascript/extractor/lib/typescript/src/typescript.js index f469e2e098dd..adc73d286fc9 100644 --- a/javascript/extractor/lib/typescript/src/typescript.js +++ b/javascript/extractor/lib/typescript/src/typescript.js @@ -1,7 +1,33 @@ -let overridePath = process.env['SEMMLE_TYPESCRIPT_HOME']; +import { createRequire } from "node:module"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; -if (overridePath != null) { - module.exports = require(overridePath); -} else { - module.exports = require('typescript'); +const overridePath = process.env["SEMMLE_TYPESCRIPT_HOME"]; +const require = createRequire(overridePath == null + ? import.meta.url + : resolve(overridePath, "package.json")); +if (overridePath != null + && require.resolve("typescript/package.json") !== require.resolve("./package.json")) { + throw new Error("SEMMLE_TYPESCRIPT_HOME must point to an installed TypeScript package with the native 7.1 API"); } +const ast = await import(pathToFileURL(require.resolve("typescript/ast")).href); +const sync = await import(pathToFileURL(require.resolve("typescript/sync")).href); + +export const { API, JsxEmit } = sync; +export const { + SyntaxKind, NodeFlags, LanguageVariant, createScanner, + isRegularExpressionLiteral, isTemplateMiddle, isTemplateTail, isTemplateExpression, + isTaggedTemplateExpression, isBinaryExpression, isVariableDeclarationList, isModuleDeclaration, + isIdentifier, isTypeParameterDeclaration, isImportAttributes, isImportAttribute, + isExportAssignment, isExportDeclaration, isImportEqualsDeclaration, isImportSpecifier, + isExportSpecifier, isImportTypeNode, isTypeReferenceNode, isHeritageClause, skipTrivia, +} = ast; +export const version = require("typescript").version; + +// Java uses symbolic names, but still expects the old EOF spelling. +export const syntaxKinds = Object.fromEntries(Object.entries(SyntaxKind) + .filter(([name]) => !/^(First|Last)/.test(name) && Number.isNaN(Number(name))) + .map(([name, value]) => [name === "EndOfFile" ? "EndOfFileToken" : name, value])); + +// This flag was removed upstream. Reserve an unused bit in the wire representation. +export const nodeFlags = { ...NodeFlags, GlobalAugmentation: 1 << 29 }; diff --git a/javascript/extractor/lib/typescript/src/virtual_source_root.ts b/javascript/extractor/lib/typescript/src/virtual_source_root.ts deleted file mode 100644 index 6ca9a580b566..000000000000 --- a/javascript/extractor/lib/typescript/src/virtual_source_root.ts +++ /dev/null @@ -1,69 +0,0 @@ -import * as pathlib from "path"; -import * as ts from "./typescript"; - -/** - * Mapping from the real source root to the virtual source root, - * a directory whose folder structure mirrors the real source root, but with `node_modules` installed. - */ -export class VirtualSourceRoot { - constructor( - public sourceRoot: string | null, - - /** - * Directory whose folder structure mirrors the real source root, but with `node_modules` installed, - * or undefined if no virtual source root exists. - */ - public virtualSourceRoot: string | null, - ) {} - - private static translate(oldRoot: string, newRoot: string, path: string) { - if (!oldRoot || !newRoot) return null; - let relative = pathlib.relative(oldRoot, path); - if (relative.startsWith('..') || pathlib.isAbsolute(relative)) return null; - return pathlib.join(newRoot, relative); - } - - /** - * Maps a path under the real source root to the corresponding path in the virtual source root. - * - * Returns `null` for paths already in the virtual source root. - */ - public toVirtualPath(path: string) { - let { virtualSourceRoot } = this; - if (path.startsWith(virtualSourceRoot)) { - // 'qltest' creates a virtual source root inside the real source root. - // Make sure such files don't appear to be inside the real source root. - return null; - } - return VirtualSourceRoot.translate(this.sourceRoot, virtualSourceRoot, path); - } - - /** - * Maps a path under the virtual source root to the corresponding path in the real source root. - */ - public fromVirtualPath(path: string) { - return VirtualSourceRoot.translate(this.virtualSourceRoot, this.sourceRoot, path); - } - - /** - * Maps a path under the real source root to the corresponding path in the virtual source root. - */ - public toVirtualPathIfFileExists(path: string) { - let virtualPath = this.toVirtualPath(path); - if (virtualPath != null && ts.sys.fileExists(virtualPath)) { - return virtualPath; - } - return null; - } - - /** - * Maps a path under the real source root to the corresponding path in the virtual source root. - */ - public toVirtualPathIfDirectoryExists(path: string) { - let virtualPath = this.toVirtualPath(path); - if (virtualPath != null && ts.sys.directoryExists(virtualPath)) { - return virtualPath; - } - return null; - } -} diff --git a/javascript/extractor/lib/typescript/test/lifecycle.test.mjs b/javascript/extractor/lib/typescript/test/lifecycle.test.mjs new file mode 100644 index 000000000000..4fef0e37f862 --- /dev/null +++ b/javascript/extractor/lib/typescript/test/lifecycle.test.mjs @@ -0,0 +1,214 @@ +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import { copyFileSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { createInterface } from "node:readline"; +import { after, test } from "node:test"; +import { setTimeout } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const directory = mkdtempSync(join(tmpdir(), "codeql-typescript-lifecycle-")); +const entryPoint = join(directory, "main.js"); +for (const name of ["main.js", "ast_extractor.js", "typescript.js"]) { + copyFileSync(join(root, "build", name), join(directory, name)); +} +copyFileSync(join(root, "package.json"), join(directory, "package.json")); +symlinkSync(join(root, "node_modules"), join(directory, "node_modules"), "dir"); +const filename = join(directory, "input.ts"); +writeFileSync(filename, "const value = 1;"); +const preload = join(directory, "track-native.cjs"); +writeFileSync(preload, ` +const cp = require("node:child_process"); +const fs = require("node:fs"); +const spawn = cp.spawn; +cp.spawn = function (...args) { + const child = spawn(...args); + fs.appendFileSync(process.env.NATIVE_LOG, JSON.stringify({ + pid: child.pid, + memoryLimit: process.env.GOMEMLIMIT, + stdio: args[2].stdio, + }) + "\\n"); + if (process.env.KILL_NATIVE) child.kill("SIGKILL"); + return child; +}; +`); +after(() => rmSync(directory, { recursive: true, force: true })); + +function options(name, env = {}) { + const environment = { ...process.env }; + delete environment.SEMMLE_TYPESCRIPT_HOME; + return { + encoding: "utf8", + timeout: 10000, + env: { + ...environment, + ...env, + NATIVE_LOG: join(directory, name + ".log"), + }, + }; +} + +function run(name, input, env = {}, args = []) { + const opts = options(name, env); + const result = spawnSync(process.execPath, ["--require", preload, entryPoint, ...args], { + ...opts, input, + }); + assert.ifError(result.error); + assert.equal(result.signal, null, result.stderr); + return result; +} + +function nativeProcesses(name) { + return readFileSync(join(directory, name + ".log"), "utf8").trim().split("\n").map(JSON.parse); +} + +async function assertTerminated(pid) { + for (let i = 0; i < 100; ++i) { + try { + process.kill(pid, 0); + } catch (error) { + assert.equal(error.code, "ESRCH"); + return; + } + await setTimeout(10); + } + assert.fail(`Native process ${pid} survived disposal`); +} + +test("compiled wrapper preserves heritage, assertions and tokens, and accepts source-phase imports", () => { + const input = join(directory, "compatibility.ts"); + writeFileSync(input, ` + interface I extends A, N.B {} + class C implements I {} + import data from "data" assert { type: "json" }; + import source wasm from "./module.wasm"; + const expression = /data/g; + const template = \`value:\${data}\`; + `); + const result = run("compatibility", [ + { command: "get-metadata" }, + { command: "parse", filename: input }, + { command: "quit" }, + ].map(command => JSON.stringify(command) + "\n").join("")); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stderr, ""); + const [metadata, { ast }] = result.stdout.trim().split("\n").map(JSON.parse); + const kinds = metadata.syntaxKinds; + assert.deepEqual(ast.parseDiagnostics, []); + const heritage = ast.statements[0].heritageClauses[0].types; + assert.equal(heritage[0].kind, kinds.ExpressionWithTypeArguments); + assert.equal(heritage[0].expression.text, "A"); + assert.equal(heritage[1].expression.right.text, "B"); + assert.equal(heritage[1].typeArguments[0].kind, kinds.StringKeyword); + assert.equal(ast.statements[2].attributes.elements[0].key.text, "type"); + assert.equal(ast.statements[2].attributes.elements[0].value.text, "json"); + assert.equal(ast.statements[3].importClause.phaseModifier, kinds.SourceKeyword); + assert.equal(ast.statements[3].importClause.name.text, "wasm"); + assert(ast.$tokens.some(token => token.text === "/data/g" && token.kind === kinds.SlashToken)); + assert(ast.$tokens.some(token => token.text === "}`" && token.kind === kinds.CloseBraceToken)); +}); + +test("compiled ESM wrapper releases each native process and budgets its memory", { timeout: 15000 }, async t => { + const name = "repeated"; + const opts = options(name, { SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD: "1000", GOMEMLIMIT: "900MiB" }); + const child = spawn(process.execPath, ["--require", preload, entryPoint], opts); + let stderr = ""; + child.stderr.setEncoding("utf8").on("data", data => { stderr += data; }); + const lines = createInterface({ input: child.stdout })[Symbol.asyncIterator](); + const exited = new Promise((accept, reject) => { + child.on("error", reject); + child.on("close", code => accept(code)); + }); + t.after(async () => { + child.stdin.destroy(); + child.kill(); + await exited; + }); + async function request(command) { + child.stdin.write(JSON.stringify(command) + "\n"); + const line = await lines.next(); + assert.equal(line.done, false, stderr); + return JSON.parse(line.value); + } + for (let i = 0; i < 10; ++i) { + const response = await request({ command: "parse", filename }); + assert.equal(response.type, "ast"); + assert.deepEqual(response.ast.parseDiagnostics, []); + const processes = nativeProcesses(name); + assert.equal(processes.length, i + 1); + const native = processes[i]; + assert(Number(native.memoryLimit) > 0); + assert(Number(native.memoryLimit) < 1000000000); + assert.deepEqual(native.stdio, process.platform === "win32" + ? ["ignore", "ignore", "inherit"] : ["pipe", "pipe", "inherit"]); + await assertTerminated(native.pid); + assert.deepEqual(await request({ command: "reset" }), { type: "reset-done" }); + } + // Quit must terminate even when the sender leaves stdin open. + child.stdin.write('{"command":"quit"}\n'); + assert.equal(await exited, 0, stderr); + assert.equal(stderr, ""); +}); + +test("reset and quit invalidate queued prefetch callbacks", () => { + const missing = join(directory, "missing.ts"); + const result = run("cancelled", [ + { command: "prepare-files", filenames: [missing] }, + { command: "reset" }, + { command: "prepare-files", filenames: [missing] }, + { command: "quit" }, + { command: "parse", filename: missing }, + ].map(command => JSON.stringify(command) + "\n").join("")); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stderr, ""); + assert.deepEqual(result.stdout.trim().split("\n").map(JSON.parse), [ + { type: "ok" }, { type: "reset-done" }, { type: "ok" }, + ]); +}); + +test("single-file mode and EOF flush responses and close native processes", async () => { + for (const [name, input, args] of [ + ["single", "", [filename]], + ["eof", JSON.stringify({ command: "parse", filename }) + "\n", []], + ]) { + const result = run(name, input, {}, args); + assert.equal(result.status, 0, result.stderr); + assert.equal(JSON.parse(result.stdout).type, "ast"); + assert.equal(result.stderr, ""); + await assertTerminated(nativeProcesses(name)[0].pid); + } +}); + +test("native failures are fatal, close the process, and never become JSON diagnostics", async () => { + const result = run("killed", "", { KILL_NATIVE: "1" }, [filename]); + assert.equal(result.status, 1); + assert.equal(result.stdout, ""); + assert.match(result.stderr, /Error|EOF|EPIPE|ECONNRESET/); + await assertTerminated(nativeProcesses("killed")[0].pid); +}); + +test("invalid commands, files, memory settings, and runtime overrides fail on stderr", () => { + for (const [name, input, env, args, message] of [ + ["json", "{\n", {}, [], /SyntaxError/], + ["command", '{"command":"invalid"}\n', {}, [], /Unknown command/], + ["missing", "", {}, [join(directory, "missing.ts")], /did not return a source file/], + ["memory", "", { SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD: "invalid" }, [filename], /MEMORY_THRESHOLD/], + ["override", "", { SEMMLE_TYPESCRIPT_HOME: directory }, [filename], /SEMMLE_TYPESCRIPT_HOME/], + ]) { + const result = run(name, input, env, args); + assert.equal(result.status, 1, name); + assert.equal(result.stdout, "", name); + assert.match(result.stderr, message); + } +}); + +test("SEMMLE_TYPESCRIPT_HOME selects the installed native package", () => { + const result = run("override-valid", "", { + SEMMLE_TYPESCRIPT_HOME: join(root, "node_modules", "typescript"), + }, [filename]); + assert.equal(result.status, 0, result.stderr); + assert.equal(JSON.parse(result.stdout).type, "ast"); + assert.equal(result.stderr, ""); +}); diff --git a/javascript/extractor/lib/typescript/test/package_test.py b/javascript/extractor/lib/typescript/test/package_test.py new file mode 100644 index 000000000000..b0feefcd94b7 --- /dev/null +++ b/javascript/extractor/lib/typescript/test/package_test.py @@ -0,0 +1,216 @@ +import json +import os +from pathlib import Path +import stat +import subprocess +import tempfile +import unittest +import zipfile + +from python.runfiles import runfiles + + +PLATFORMS = { + "linux64": ["linux-x64"], + "linux-arm64": ["linux-arm64"], + "osx64": ["darwin-arm64", "darwin-x64"], + "win64": ["win32-x64"], +} + + +class PackageTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + files = runfiles.Create() + cls.node = files.Rlocation(os.environ["NODE_BIN"]) + cls.ripunzip = files.Rlocation(os.environ["RIPUNZIP"]) + cls.script = files.Rlocation(os.environ["PACKAGE_RUNTIME"]) + cls.wrapper_zip = files.Rlocation(os.environ["TS_WRAPPER_ZIP"]) + cls.native_zip = files.Rlocation(os.environ["TS_NATIVE_ZIP"]) + cls.common_zip = files.Rlocation(os.environ["TS_COMMON_ZIP"]) + cls.arch_zip = files.Rlocation(os.environ["TS_ARCH_ZIP"]) + with zipfile.ZipFile(cls.wrapper_zip) as archive: + cls.version = json.loads( + archive.read("typescript-parser-wrapper/package.json") + )["dependencies"]["typescript"] + + def stage(self, directory, distribution, mismatch=False): + modules = directory / "node_modules" + typescript = modules / "typescript" + typescript.mkdir(parents=True) + dependencies = {} + for platform in {p for ps in PLATFORMS.values() for p in ps}: + name = "typescript-" + platform + package = modules / "@typescript" / name + (package / "lib").mkdir(parents=True) + manifest = { + "name": "@typescript/" + name, + "version": "invalid" if mismatch else self.version, + } + (package / "package.json").write_text(json.dumps(manifest)) + executable = package / "lib" / ( + "tsc.exe" if platform.startswith("win32-") else "tsc" + ) + executable.write_text("native executable fixture") + executable.chmod(0o644) + dependencies[manifest["name"]] = self.version + (typescript / "package.json").write_text( + json.dumps({"version": self.version, "optionalDependencies": dependencies}) + ) + packages = subprocess.run( + [self.node, self.script, "--packages", distribution], + cwd=directory, + capture_output=True, + text=True, + timeout=30, + ) + if distribution in PLATFORMS: + self.assertEqual(packages.returncode, 0, packages.stderr) + self.assertEqual( + packages.stdout.split(), + [ + "@typescript/typescript-" + platform + "@" + self.version + for platform in PLATFORMS[distribution] + ], + ) + else: + self.assertNotEqual(packages.returncode, 0) + self.assertIn("Unsupported CodeQL distribution", packages.stderr) + return subprocess.run( + [self.node, self.script, distribution], + cwd=directory, + capture_output=True, + text=True, + timeout=30, + ) + + def test_distribution_selection_is_independent_of_host(self): + for distribution, platforms in PLATFORMS.items(): + with ( + self.subTest(distribution=distribution), + tempfile.TemporaryDirectory() as temp, + ): + directory = Path(temp) + result = self.stage(directory, distribution) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "") + self.assertEqual( + sorted(p.name for p in (directory / "native").iterdir()), + sorted("typescript-" + p for p in platforms), + ) + self.assertFalse((directory / "node_modules/@typescript").exists()) + if os.name != "nt": + for executable in (directory / "native").glob("*/lib/tsc"): + self.assertEqual(stat.S_IMODE(executable.stat().st_mode), 0o755) + + def test_invalid_distribution_and_mismatched_runtime_fail(self): + for distribution, mismatch, message in [ + ("unsupported", False, "Unsupported CodeQL distribution"), + ("osx64", True, "Native runtime does not match TypeScript"), + ]: + with ( + self.subTest(distribution=distribution), + tempfile.TemporaryDirectory() as temp, + ): + result = self.stage(Path(temp), distribution, mismatch) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + self.assertIn(message, result.stderr) + + def extract(self, filename, destination): + destination.mkdir(parents=True, exist_ok=True) + result = subprocess.run( + [self.ripunzip, "unzip-file", filename, "-d", str(destination)], + capture_output=True, + text=True, + timeout=60, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_installed_archives_parse_without_external_typescript(self): + host = subprocess.check_output( + [self.node, "-p", "process.platform + '-' + process.arch"], text=True + ).strip() + platforms = next(ps for ps in PLATFORMS.values() if host in ps) + with zipfile.ZipFile(self.wrapper_zip) as archive: + self.assertFalse(any("/@typescript/" in n for n in archive.namelist())) + with zipfile.ZipFile(self.native_zip) as archive: + names = archive.namelist() + self.assertEqual( + sorted({n.removeprefix("./").split("/")[0] for n in names}), + sorted("typescript-" + p for p in platforms), + ) + for platform in platforms: + prefix = "typescript-" + platform + manifest = json.loads(archive.read(prefix + "/package.json")) + self.assertEqual(manifest["version"], self.version) + binary = prefix + "/lib/" + ( + "tsc.exe" if platform.startswith("win32-") else "tsc" + ) + self.assertGreater(archive.getinfo(binary).file_size, 0) + if not platform.startswith("win32-"): + self.assertEqual( + archive.getinfo(binary).external_attr >> 16 & 0o777, 0o755 + ) + + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + with zipfile.ZipFile(self.common_zip) as archive: + self.assertFalse(any("/@typescript/" in n for n in archive.namelist())) + self.assertIn( + "tools/typescript-parser-wrapper/main.js", archive.namelist() + ) + with zipfile.ZipFile(self.arch_zip) as archive: + self.assertEqual( + sorted(n for n in archive.namelist() if not n.endswith("/")), + sorted( + "tools/typescript-parser-wrapper/node_modules/@typescript/" + n + for n in names + if not n.endswith("/") + ), + ) + self.extract(self.common_zip, directory) + self.extract(self.arch_zip, directory) + wrapper = directory / "tools/typescript-parser-wrapper" + environment = dict(os.environ) + environment.pop("SEMMLE_TYPESCRIPT_HOME", None) + environment.pop("NODE_PATH", None) + commands = [] + for name, source in [ + ("valid.ts", "const value: number = 1;"), + ("valid.tsx", "const value =
Hello
;"), + ("invalid.ts", "const value = ;"), + ]: + filename = directory / name + filename.write_text(source) + commands.extend( + [ + {"command": "parse", "filename": str(filename)}, + {"command": "reset"}, + ] + ) + commands.append({"command": "quit"}) + result = subprocess.run( + [self.node, str(wrapper / "main.js")], + input="".join(json.dumps(command) + "\n" for command in commands), + cwd=directory, + env=environment, + capture_output=True, + text=True, + timeout=60, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stderr, "") + responses = [json.loads(line) for line in result.stdout.splitlines()] + self.assertEqual(len(responses), 6) + for response in responses[::2]: + self.assertEqual(response["type"], "ast") + for response in responses[1::2]: + self.assertEqual(response, {"type": "reset-done"}) + self.assertEqual(responses[0]["ast"]["parseDiagnostics"], []) + self.assertEqual(responses[2]["ast"]["parseDiagnostics"], []) + self.assertTrue(responses[4]["ast"]["parseDiagnostics"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/javascript/extractor/lib/typescript/ts7/package-lock.json b/javascript/extractor/lib/typescript/ts7/package-lock.json new file mode 100644 index 000000000000..a1fb5cd23ca1 --- /dev/null +++ b/javascript/extractor/lib/typescript/ts7/package-lock.json @@ -0,0 +1,163 @@ +{ + "name": "codeql-typescript-7-parser-probe", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "codeql-typescript-7-parser-probe", + "version": "0.0.0", + "dependencies": { + "typescript": "7.1.0-dev.20261015.1", + "typescript-legacy": "npm:typescript@5.9.2" + } + }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-cIBrgsPrcPJfnyEP1gnonNoBcV6WZAxpWdrOO6F60eDhwknkHPxcTmdwuFA9ar07fFzyfp7XhKRrqz7OlrSQnA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-TkxvjvYuMDSzEmeI3g6XzLPKb6FBlonWu9FkmLMeXMNAxWRBjJmXdyKEBhDM3EASDBXAZL68X2sEim1l7e9r4g==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-8hnNa5ptGrsXCl5bs7ijxVQsuOcaCGxeDOAWqFdCNIRDQ9Z69NNrmizH650gPnZQCcNyxkFpK7QmoGSv9jsHSg==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-LaWnTy/FahvhOdo8w2ugYkDAmbcBuY6Zm8jNKYtqghx8bGeH0Mj+dX4533++tzPxAZoRnCE0WdGoBfzpDlLS8Q==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-9hRSH3i84yqjpVZtapYH/GAvRxTnh71Bui6EQDWkvrBZ6wvFleEgsyEg9RXxQDedXNMMHwKRGSYT7NvRymuytw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-hUdo1fnWfYsC4Az+1uBySKCaLBvrfWEH+jALFFvY5fROBU7cqoobVRQmOweJpiLsUQDqrHXAD3v0rnCEhCKI9w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-SQephnLxrS46fwPsM2K1LGbD8jHT4H4KF+bBoGnaSoEMLgLulAdrhY7A7lWUI3OMRuwawjfHxsnSV4TjeKEhcw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/typescript": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-IonOVC0CPC24brjf3BOPYSrU8CioR/p15jRRUnx6M6zywu4tcDw0vcIKE1dR9XpOEpArd15xZ5rIFj9wjAyOkg==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc" + }, + "engines": { + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-darwin-arm64": "7.1.0-dev.20261015.1", + "@typescript/typescript-darwin-x64": "7.1.0-dev.20261015.1", + "@typescript/typescript-linux-arm": "7.1.0-dev.20261015.1", + "@typescript/typescript-linux-arm64": "7.1.0-dev.20261015.1", + "@typescript/typescript-linux-x64": "7.1.0-dev.20261015.1", + "@typescript/typescript-win32-arm64": "7.1.0-dev.20261015.1", + "@typescript/typescript-win32-x64": "7.1.0-dev.20261015.1" + } + }, + "node_modules/typescript-legacy": { + "name": "typescript", + "version": "5.9.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.2.tgz", + "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + } + } +} diff --git a/javascript/extractor/lib/typescript/ts7/package.json b/javascript/extractor/lib/typescript/ts7/package.json new file mode 100644 index 000000000000..52fc3816eb14 --- /dev/null +++ b/javascript/extractor/lib/typescript/ts7/package.json @@ -0,0 +1,13 @@ +{ + "name": "codeql-typescript-7-parser-probe", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "test": "node --test probe.mjs wrapper.test.mjs" + }, + "dependencies": { + "typescript": "7.1.0-dev.20261015.1", + "typescript-legacy": "npm:typescript@5.9.2" + } +} diff --git a/javascript/extractor/lib/typescript/ts7/probe.mjs b/javascript/extractor/lib/typescript/ts7/probe.mjs new file mode 100644 index 000000000000..5a334987341f --- /dev/null +++ b/javascript/extractor/lib/typescript/ts7/probe.mjs @@ -0,0 +1,260 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { after, before, test } from "node:test"; +import { pathToFileURL } from "node:url"; + +const require = createRequire(import.meta.url); +const version = require("./package.json").dependencies.typescript; +const { API, JsxEmit } = await import(pathToFileURL(require.resolve("typescript/sync"))); +const { SyntaxKind, ScriptKind, LanguageVariant, createScanner } = + await import(pathToFileURL(require.resolve("typescript/ast"))); +const legacy = require("typescript-legacy"); +const compilerOptions = { noResolve: true, noLib: true, jsx: JsxEmit.Preserve, experimentalDecorators: true }; +const directory = mkdtempSync(join(tmpdir(), "codeql-typescript-7-probe-")); +let api; + +before(() => { + api = new API({ cwd: directory }); + api.parseCommandLine([]); +}); +after(() => { + try { + api?.close(); + } finally { + rmSync(directory, { recursive: true }); + } +}); + +function withProgram(filename, text, callback) { + const path = join(directory, filename); + writeFileSync(path, text); + const program = api.createProgram([path], compilerOptions); + try { + assert.deepEqual(program.getSourceFileNames(), [path]); + const file = program.getSourceFile(path); + assert(file, "The program must contain the input file"); + assert.equal(file.text, text); + return callback(file, program); + } finally { + program.dispose(); + } +} + +function kindName(kind, kinds) { + const entry = Object.entries(kinds).find(([name, value]) => value === kind && !/^(First|Last)/.test(name)); + assert(entry, `Missing syntax kind ${kind}`); + return entry[0] + .replace(/^EndOfFile$/, "EndOfFileToken") + .replace(/^TypeOf/, "Typeof") + .replace(/^InstanceOf/, "Instanceof") + .replace(/^KeyOf/, "Keyof"); +} + +function flatten(file, kinds, visitChildren) { + const nodes = []; + function visit(node) { + nodes.push([kindName(node.kind, kinds), node.pos, node.end, node.getStart(file)]); + visitChildren(node, visit); + } + visit(file); + return nodes; +} + +test("the installed API and native CLI are the pinned npm next build", () => { + assert.match(version, /^7\.1\.0-dev\./); + assert.equal(require("typescript").version, version); + assert.equal(require("typescript-legacy").version, "5.9.2"); + const cli = join(dirname(require.resolve("typescript/package.json")), "bin/tsc"); + assert.equal(execFileSync(process.execPath, [cli, "--version"], { encoding: "utf8" }).trim(), `Version ${version}`); + assert.equal(api.getCurrentDirectory(), directory); +}); + +test("TypeScript AST traversal, parent links and UTF-16 positions match 5.9", () => { + const text = "// header\r\nexport const emoji = \"\u{1f600}\"; const answer: number = 42;\r\n"; + withProgram("valid.ts", text, (file, program) => { + assert.equal(file.scriptKind, ScriptKind.TS); + assert.deepEqual(program.getSyntacticDiagnostics(), []); + const previous = legacy.createSourceFile("valid.ts", text, legacy.ScriptTarget.Latest, true); + assert.deepEqual( + flatten(file, SyntaxKind, (node, visit) => node.forEachChild(visit)), + flatten(previous, legacy.SyntaxKind, legacy.forEachChild) + ); + const declaration = file.statements[0].declarationList.declarations[0]; + assert.equal(declaration.parent, file.statements[0].declarationList); + assert.equal(declaration.getSourceFile(), file); + assert.equal(declaration.name.getText(), "emoji"); + assert.deepEqual(file.getLineStarts(), previous.getLineStarts()); + const answer = text.indexOf("answer"); + assert.deepEqual( + file.getLineAndCharacterOfPosition(answer), + previous.getLineAndCharacterOfPosition(answer) + ); + assert.equal( + file.getPositionOfLineAndCharacter(1, answer - text.indexOf("export")), + answer + ); + }); +}); + +test("TSX is parsed as JSX without resolving imports or loading libraries", () => { + const text = "import { missing } from './missing';\nexport const element =
{missing}
;"; + withProgram("component.tsx", text, (file, program) => { + assert.equal(file.scriptKind, ScriptKind.TSX); + assert.deepEqual(program.getSyntacticDiagnostics(), []); + const element = file.statements[1].declarationList.declarations[0].initializer; + assert.equal(element.kind, SyntaxKind.JsxElement); + assert.equal(element.openingElement.tagName.getText(), "div"); + const previous = legacy.createSourceFile("component.tsx", text, legacy.ScriptTarget.Latest, true); + assert.deepEqual( + flatten(file, SyntaxKind, (node, visit) => node.forEachChild(visit)), + flatten(previous, legacy.SyntaxKind, legacy.forEachChild) + ); + }); +}); + +test("syntactic diagnostics preserve 5.9 messages and UTF-16 offsets", () => { + const text = "const emoji = \"\u{1f600}\";\r\nconst broken = ;\r\n"; + withProgram("malformed.ts", text, (file, program) => { + const diagnostics = program.getSyntacticDiagnostics(); + assert(diagnostics.length > 0); + const previous = legacy.createSourceFile("malformed.ts", text, legacy.ScriptTarget.Latest, true); + assert.deepEqual( + diagnostics.map(d => [d.code, d.text, d.pos, d.end - d.pos]), + previous.parseDiagnostics.map(d => [d.code, d.messageText, d.start, d.length]) + ); + for (const diagnostic of diagnostics) { + assert.deepEqual(diagnostic.startPosition, file.getLineAndCharacterOfPosition(diagnostic.pos)); + } + }); +}); + +test("the scanner preserves token text and positions, including regex rescanning", () => { + const text = "// comment\r\nconst pattern = /a+/gi; const result = 8 >> 1;"; + const scanner = createScanner(false, LanguageVariant.Standard, text); + const previous = legacy.createScanner(legacy.ScriptTarget.Latest, false, legacy.LanguageVariant.Standard, text); + const tokens = []; + const previousTokens = []; + for (;;) { + let kind = scanner.scan(); + if (kind === SyntaxKind.SlashToken) kind = scanner.reScanSlashToken(); + if (kind === SyntaxKind.GreaterThanToken) kind = scanner.reScanGreaterToken(); + tokens.push([kindName(kind, SyntaxKind), scanner.getTokenStart(), scanner.getTokenText()]); + if (kind === SyntaxKind.EndOfFile) break; + } + for (;;) { + let kind = previous.scan(); + if (kind === legacy.SyntaxKind.SlashToken) kind = previous.reScanSlashToken(); + if (kind === legacy.SyntaxKind.GreaterThanToken) kind = previous.reScanGreaterToken(); + previousTokens.push([kindName(kind, legacy.SyntaxKind), previous.getTokenPos(), previous.getTokenText()]); + if (kind === legacy.SyntaxKind.EndOfFileToken) break; + } + assert.deepEqual(tokens, previousTokens); + assert(tokens.some(([kind]) => kind === "RegularExpressionLiteral")); + assert(tokens.some(([kind]) => kind === "GreaterThanGreaterThanToken")); +}); + +test("template-tail rescanning preserves tagged and untagged template tokens", () => { + for (const tagged of [false, true]) { + const text = (tagged ? "tag" : "") + "`before ${value} after`"; + const scanner = createScanner(false, LanguageVariant.Standard, text); + const previous = legacy.createScanner(legacy.ScriptTarget.Latest, false, legacy.LanguageVariant.Standard, text); + const tokens = []; + const previousTokens = []; + for (;;) { + let kind = scanner.scan(); + if (kind === SyntaxKind.CloseBraceToken) kind = scanner.reScanTemplateToken(tagged); + tokens.push([kindName(kind, SyntaxKind), scanner.getTokenStart(), scanner.getTokenText()]); + if (kind === SyntaxKind.EndOfFile) break; + } + for (;;) { + let kind = previous.scan(); + if (kind === legacy.SyntaxKind.CloseBraceToken) kind = previous.reScanTemplateToken(tagged); + previousTokens.push([kindName(kind, legacy.SyntaxKind), previous.getTokenPos(), previous.getTokenText()]); + if (kind === legacy.SyntaxKind.EndOfFileToken) break; + } + assert.deepEqual(tokens, previousTokens); + assert(tokens.some(([kind]) => kind === "TemplateTail")); + } +}); + +test("AST getters require explicit projection before JSON serialization", () => { + withProgram("projection.ts", "const value = 1;", file => { + assert.equal(Object.hasOwn(file, "kind"), false); + const node = file.statements[0].declarationList.declarations[0].name; + const projected = JSON.parse(JSON.stringify({ + kind: node.kind, + text: node.text, + $pos: node.getStart(file), + $end: node.end + })); + assert.deepEqual(projected, { + kind: SyntaxKind.Identifier, + text: "value", + $pos: 6, + $end: 11 + }); + }); +}); + +test("standalone source-file leases can be released and recreated", () => { + const path = join(directory, "standalone.ts"); + const retained = api.createSourceFile(path, "const value = 1;"); + try { + assert.equal(retained.sourceFile.statements.length, 1); + assert.equal(retained.sourceFile.statements[0].parent, retained.sourceFile); + } finally { + retained.dispose(); + } + retained.dispose(); + const next = api.createSourceFile(path, "const value = 2;"); + try { + assert.equal(next.sourceFile.statements[0].declarationList.declarations[0].initializer.text, "2"); + } finally { + next.dispose(); + } +}); + +test("program disposal and cache clearing allow repeated independent parses", () => { + for (let index = 0; index < 20; ++index) { + withProgram("repeated.ts", `const value = ${index};`, (file, program) => { + assert.equal(file.statements[0].declarationList.declarations[0].initializer.text, String(index)); + assert.deepEqual(program.getSyntacticDiagnostics(), []); + }); + api.clearSourceFileCache(); + } +}); + +test("disposing a program releases its native snapshot", () => { + const path = join(directory, "released.ts"); + writeFileSync(path, "const value = 1;"); + const program = api.createProgram([path], compilerOptions); + try { + assert(program.getSourceFile(path)); + } finally { + program.dispose(); + } + program.dispose(); + assert.throws(() => program.getSourceFileNames(), /snapshot \d+ not found/i); +}); + +test("an API can be closed explicitly and replaced", () => { + const other = new API({ cwd: directory }); + try { + const retained = other.createSourceFile(join(directory, "close.ts"), "const value = 1;"); + retained.dispose(); + } finally { + other.close(); + } + assert.throws(() => other.createSourceFile(join(directory, "closed.ts"), ""), /closed/i); + const replacement = new API({ cwd: directory }); + try { + replacement.parseCommandLine([]); + assert.equal(replacement.getCurrentDirectory(), directory); + } finally { + replacement.close(); + } +}); diff --git a/javascript/extractor/lib/typescript/ts7/wrapper.test.mjs b/javascript/extractor/lib/typescript/ts7/wrapper.test.mjs new file mode 100644 index 000000000000..d56c1fe2e095 --- /dev/null +++ b/javascript/extractor/lib/typescript/ts7/wrapper.test.mjs @@ -0,0 +1,292 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawn } from "node:child_process"; +import { copyFileSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { createInterface } from "node:readline"; +import { after, test } from "node:test"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { API, JsxEmit } from "typescript/sync"; +import { SyntaxKind, NodeFlags } from "typescript/ast"; + +const require = createRequire(import.meta.url); +const legacy = require("typescript-legacy"); +const here = dirname(fileURLToPath(import.meta.url)); +const directory = mkdtempSync(join(tmpdir(), "codeql-typescript-wrapper-")); +const entryPoint = join(directory, "main.js"); +writeFileSync(join(directory, "package.json"), '{"type":"module"}'); +symlinkSync(join(here, "node_modules"), join(directory, "node_modules"), "dir"); +copyFileSync(join(here, "../src/typescript.js"), join(directory, "typescript.js")); +for (const name of ["main", "ast_extractor"]) { + const source = readFileSync(join(here, `../src/${name}.ts`), "utf8"); + writeFileSync(join(directory, `${name}.js`), legacy.transpileModule(source, { + compilerOptions: { target: legacy.ScriptTarget.ES2022, module: legacy.ModuleKind.ES2022 }, + }).outputText); +} +const { extractAst } = await import(pathToFileURL(join(directory, "ast_extractor.js"))); +const { syntaxKinds, nodeFlags } = await import(pathToFileURL(join(directory, "typescript.js"))); +const api = new API(); + +after(() => { + try { + api.close(); + } finally { + rmSync(directory, { recursive: true }); + } +}); + +function extract(text, extension = ".ts") { + const filename = join(directory, "input" + extension); + writeFileSync(filename, text); + const program = api.createProgram([filename], { + noResolve: true, noLib: true, jsx: JsxEmit.Preserve, experimentalDecorators: true, + }); + try { + return JSON.parse(JSON.stringify(extractAst(program.getSourceFile(filename), program.getSyntacticDiagnostics()))); + } finally { + program.dispose(); + api.clearSourceFileCache(); + } +} + +function declaration(ast, index = 0) { + return ast.statements[index].declarationList.declarations[0]; +} + +test("wire metadata preserves the Java converter's syntax names and flags", () => { + assert.equal(syntaxKinds.EndOfFileToken, SyntaxKind.EndOfFile); + assert(!("EndOfFile" in syntaxKinds)); + assert.equal(syntaxKinds.TypeOfExpression, SyntaxKind.TypeOfExpression); + assert.equal(syntaxKinds.TypeOfKeyword, SyntaxKind.TypeOfKeyword); + assert.equal(syntaxKinds.InstanceOfKeyword, SyntaxKind.InstanceOfKeyword); + assert.equal(syntaxKinds.KeyOfKeyword, SyntaxKind.KeyOfKeyword); + assert.equal(nodeFlags.NestedNamespace, NodeFlags.NestedNamespace); + assert.equal(nodeFlags.GlobalAugmentation & Object.values(NodeFlags) + .filter(value => typeof value === "number").reduce((flags, value) => flags | value, 0), 0); +}); + +test("AST getters are projected without native implementation details or unrelated booleans", () => { + const text = "// comment\r\nexport const \\u0061 = \"\u{1f600}\";\r\nlet b = 2; var c = 3;"; + const ast = extract(text); + const first = declaration(ast); + assert.equal(first.name.text, "a"); + assert.equal(first.name.$pos, text.indexOf("\\u0061")); + assert.equal(first.initializer.text, "\u{1f600}"); + assert.equal(first.initializer.$end, text.indexOf('";') + 1); + assert.equal(ast.statements[0].modifiers[0].kind, SyntaxKind.ExportKeyword); + assert.deepEqual(ast.$lineStarts, [0, text.indexOf("export"), text.indexOf("let")]); + assert.deepEqual(ast.statements.map(statement => statement.declarationList.$declarationKind), ["const", "let", "var"]); + assert.deepEqual(ast.parseDiagnostics, []); + const json = JSON.stringify(ast); + for (const field of ["parent", "view", "sourceFile", "isTypeOnly", "isTypeOf", "isExportEquals"]) { + assert(!json.includes(`"${field}":`), field); + } +}); + +test("syntax error messages and UTF-16 offsets match the old Java-facing response", () => { + const text = 'const emoji = "\u{1f600}";\r\nconst broken = ;\r\n'; + const ast = extract(text); + const previous = legacy.createSourceFile("input.ts", text, legacy.ScriptTarget.Latest, true); + assert.deepEqual(ast.parseDiagnostics, previous.parseDiagnostics.map(diagnostic => ({ + messageText: diagnostic.messageText, $pos: diagnostic.start, + }))); + assert.deepEqual(ast.$lineStarts, previous.getLineStarts()); +}); + +test("optional members, definite assignment, type defaults, and decorators keep their old fields", () => { + const ast = extract(` + type T = U; + interface I { p?: string; m?(): void } + @sealed export class C { + @field p!: string; + @method m(@parameter publicArg?: number) {} + } + `); + assert.deepEqual(ast.parseDiagnostics, []); + assert.equal(ast.statements[0].typeParameters[0].default.kind, SyntaxKind.NumberKeyword); + const [property, method] = ast.statements[1].members; + assert.equal(property.questionToken.kind, SyntaxKind.QuestionToken); + assert.equal(method.questionToken.kind, SyntaxKind.QuestionToken); + const cls = ast.statements[2]; + assert.deepEqual(cls.modifiers.map(modifier => modifier.kind), [SyntaxKind.Decorator, SyntaxKind.ExportKeyword]); + assert.equal(cls.members[0].exclamationToken.kind, SyntaxKind.ExclamationToken); + assert.equal(cls.members[0].modifiers[0].kind, SyntaxKind.Decorator); + assert.equal(cls.members[1].parameters[0].modifiers[0].kind, SyntaxKind.Decorator); + assert.equal(cls.members[1].parameters[0].questionToken.kind, SyntaxKind.QuestionToken); +}); + +test("global augmentations and dotted namespaces retain their Java node classifications", () => { + const ast = extract(` + declare /* a */ global { interface Window {} } + namespace global {} + module global {} + namespace A.B.C {} + namespace A { namespace B {} } + declare module "external" {} + `); + assert.deepEqual(ast.parseDiagnostics, []); + const [global, namedGlobal, moduleGlobal, dotted, nested, external] = ast.statements; + assert(global.flags & nodeFlags.GlobalAugmentation); + for (const node of [namedGlobal, moduleGlobal, dotted, nested, external]) { + assert.equal(node.flags & nodeFlags.GlobalAugmentation, 0); + } + assert(dotted.body.flags & nodeFlags.NestedNamespace); + assert(dotted.body.body.flags & nodeFlags.NestedNamespace); + assert.equal(nested.body.statements[0].flags & nodeFlags.NestedNamespace, 0); + assert.equal(external.name.text, "external"); +}); + +test("import attributes and type-only phases preserve the converter's property names", () => { + const ast = extract(` + import type { X } from "types"; + import { type Y, Z } from "types"; + import data from "data" with { type: "json" }; + export { data } from "data" with { type: "json" }; + type T = typeof import("data", { with: { type: "json" } }); + export type { X }; + `); + assert.deepEqual(ast.parseDiagnostics, []); + assert.equal(ast.statements[0].importClause.phaseModifier, SyntaxKind.TypeKeyword); + assert.equal(ast.statements[1].importClause.namedBindings.elements[0].isTypeOnly, true); + assert.equal(ast.statements[1].importClause.namedBindings.elements[1].isTypeOnly, false); + for (const node of [ast.statements[2], ast.statements[3], ast.statements[4].type]) { + assert.equal(node.attributes.elements[0].key.text, "type"); + assert.equal(node.attributes.elements[0].value.text, "json"); + } + assert.equal(ast.statements[4].type.isTypeOf, true); + assert.equal(ast.statements[5].isTypeOnly, true); +}); + +test("regex, nested templates, tagged templates, type templates and operators are rescanned", () => { + const text = 'const x = `a${/a+/gi.test(s) ? tag`b${v}c` : `d${w}e`}f`; ' + + 'type T = `a${string}b`; const y = a >= b >> c >>> d; a >>= b; a >>>= b;'; + const ast = extract(text); + assert.deepEqual(ast.parseDiagnostics, []); + const rescannedTexts = ["/a+/gi", "}c`", "}e`", "}f`", "}b`", ">=", ">>", ">>>", ">>=", ">>>="]; + const tokens = ast.$tokens.filter(token => rescannedTexts.includes(token.text)); + assert.deepEqual(tokens.map(token => token.text), ["/a+/gi", "}c`", "}e`", "}f`", "}b`", ">=", ">>", ">>>", ">>=", ">>>="]); + assert.deepEqual(tokens.map(token => token.kind), [ + SyntaxKind.SlashToken, + ...Array(4).fill(SyntaxKind.CloseBraceToken), + ...Array(5).fill(SyntaxKind.GreaterThanToken), + ]); + for (const token of tokens) { + assert.equal(text.slice(token.tokenPos, token.tokenPos + token.text.length), token.text); + } + assert.equal(ast.$tokens.at(-1).kind, SyntaxKind.EndOfFile); + assert.equal(ast.$tokens.at(-1).tokenPos, text.length); +}); + +test("interface heritage keeps the expression field used by the Java converter", () => { + const ast = extract(` + interface I extends A, N.B {} + class C extends Base implements A, N.B {} + `); + assert.deepEqual(ast.parseDiagnostics, []); + const [iface, cls] = ast.statements; + for (const heritage of [iface.heritageClauses[0], cls.heritageClauses[1]]) { + assert.equal(heritage.types[0].kind, SyntaxKind.ExpressionWithTypeArguments); + assert.equal(heritage.types[0].expression.text, "A"); + assert.equal(heritage.types[1].expression.right.text, "B"); + assert.equal(heritage.types[1].typeArguments[0].kind, SyntaxKind.StringKeyword); + assert(!("typeName" in heritage.types[1])); + } + assert.equal(cls.heritageClauses[0].types[0].expression.text, "Base"); +}); + +test("legacy import assertions retain attributes without hiding genuine syntax errors", () => { + const source = ` + import data from "data" assert { type: "json" }; + export { data } from "data" assert { type: "json" }; + const pattern = /data/g; + `; + const ast = extract(source); + assert.deepEqual(ast.parseDiagnostics, []); + for (const node of ast.statements.slice(0, 2)) { + assert.equal(node.attributes.elements[0].key.text, "type"); + assert.equal(node.attributes.elements[0].value.text, "json"); + } + assert(ast.$tokens.some(token => token.kind === SyntaxKind.SlashToken && token.text === "/data/g")); + const invalid = extract(source + "const broken = ;"); + assert.equal(invalid.parseDiagnostics.length, 1); + assert.equal(invalid.parseDiagnostics[0].messageText, "Expression expected."); + assert.equal(invalid.parseDiagnostics[0].$pos, (source + "const broken = ;").lastIndexOf(";")); +}); + +test("TSX children, attributes, and positions survive projection", () => { + const ast = extract('const view = <>
{value}
;', ".tsx"); + assert.deepEqual(ast.parseDiagnostics, []); + const fragment = declaration(ast).initializer; + assert.equal(fragment.kind, SyntaxKind.JsxFragment); + assert.equal(fragment.children[0].openingElement.attributes.properties[0].name.text, "title"); + assert.equal(fragment.children[0].children[0].expression.text, "value"); + assert.equal(fragment.children[1].kind, SyntaxKind.JsxSelfClosingElement); + assert.equal(fragment.children[1].attributes.properties[0].expression.text, "props"); +}); + +test("the actual wrapper handles metadata, prepared files, out-of-order parsing, reset and quit", async () => { + const first = join(directory, "first.ts"); + const second = join(directory, "second.ts"); + writeFileSync(first, "const first = 1;"); + writeFileSync(second, "let second = 2;"); + const child = spawn(process.execPath, [entryPoint], { stdio: ["pipe", "pipe", "pipe"] }); + let stderr = ""; + const lines = createInterface({ input: child.stdout })[Symbol.asyncIterator](); + child.stderr.setEncoding("utf8").on("data", data => { stderr += data; }); + const exited = new Promise((accept, reject) => { + child.on("error", reject); + child.on("close", (code, signal) => { + try { + assert.equal(signal, null); + assert.equal(code, 0, stderr); + accept(); + } catch (error) { + reject(error); + } + }); + }); + async function request(command) { + child.stdin.write(JSON.stringify(command) + "\n"); + const line = await lines.next(); + assert.equal(line.done, false, stderr); + return JSON.parse(line.value); + } + try { + const metadata = await request({ command: "get-metadata" }); + assert.equal(metadata.type, "metadata"); + assert.deepEqual(metadata.syntaxKinds, syntaxKinds); + assert.deepEqual(await request({ command: "prepare-files", filenames: [first, second] }), { type: "ok" }); + assert.equal(declaration((await request({ command: "parse", filename: first })).ast).name.text, "first"); + assert.equal(declaration((await request({ command: "parse", filename: second })).ast).name.text, "second"); + await request({ command: "prepare-files", filenames: [first, second] }); + assert.equal(declaration((await request({ command: "parse", filename: second })).ast).name.text, "second"); + assert.deepEqual(await request({ command: "reset" }), { type: "reset-done" }); + assert.equal(declaration((await request({ command: "parse", filename: first })).ast).name.text, "first"); + child.stdin.end('{"command":"quit"}\n'); + await exited; + assert.equal(stderr, ""); + } finally { + child.stdin.destroy(); + child.kill(); + await exited; + } +}); + +test("single-file invocation flushes JSON, supports the runtime override, and reports missing files", () => { + const filename = join(directory, "command-line.ts"); + writeFileSync(filename, "const value = 1;"); + const options = { + encoding: "utf8", + env: { ...process.env, SEMMLE_TYPESCRIPT_HOME: dirname(require.resolve("typescript/package.json")) }, + }; + const response = JSON.parse(execFileSync(process.execPath, [entryPoint, filename], options)); + assert.equal(declaration(response.ast).name.text, "value"); + assert.throws(() => execFileSync(process.execPath, [entryPoint, resolve(directory, "missing.ts")], { + ...options, stdio: ["ignore", "pipe", "pipe"], + }), error => { + assert.equal(error.stdout, ""); + assert.match(error.stderr, /TypeScript did not return a source file/); + return true; + }); +}); diff --git a/javascript/extractor/lib/typescript/tsconfig.json b/javascript/extractor/lib/typescript/tsconfig.json index 81038684f405..e5581708195b 100644 --- a/javascript/extractor/lib/typescript/tsconfig.json +++ b/javascript/extractor/lib/typescript/tsconfig.json @@ -1,12 +1,13 @@ { "compilerOptions": { - "module": "commonjs", + "module": "nodenext", "noImplicitAny": true, "removeComments": true, "sourceMap": false, - "baseUrl": "./", - "lib": ["es2016"], - "target": "es5", + "lib": ["es2022", "esnext.disposable"], + "types": ["node"], + "target": "es2022", + "rootDir": "src", "outDir": "build", "allowJs": true, }, diff --git a/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java b/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java index 09f05b2d9e0b..946e2200b46e 100644 --- a/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java +++ b/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java @@ -369,12 +369,7 @@ private JsonObject talkToParserWrapper(JsonObject request) { throw getExceptionFromMalformedResponse(response, e); } } catch (IOException e) { - throw new CatastrophicError( - "Could not communicate with TypeScript parser wrapper " - + "(command: ." - + parserWrapperCommand - + ").", - e); + throw getExceptionFromMalformedResponse(response, e); } } @@ -401,6 +396,14 @@ private RuntimeException getExceptionFromMalformedResponse(String response, Exce } catch (InterruptedException e1) { Exceptions.ignore(e, "This is for diagnostic purposes only."); } + if (e instanceof IOException) { + return new CatastrophicError( + "Could not communicate with TypeScript parser wrapper " + + "(command: ." + + parserWrapperCommand + + ").", + e); + } if (response == null) { return new CatastrophicError("No response from TypeScript parser wrapper", e); } diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java b/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java index 4061f1a8651b..e26747fdee95 100644 --- a/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java +++ b/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java @@ -2,13 +2,14 @@ import com.google.devtools.build.runfiles.Runfiles; import com.semmle.util.process.Env; -import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.Map; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -36,20 +37,10 @@ public static void setUp() throws Exception { Runfiles.Preloaded runfiles = Runfiles.preload(); String nodePath = runfiles.unmapped().rlocation(System.getenv("NODE_BIN")); String tsWrapperZip = runfiles.unmapped().rlocation(System.getenv("TS_WRAPPER_ZIP")); + String tsNativeZip = runfiles.unmapped().rlocation(System.getenv("TS_NATIVE_ZIP")); Path tempDir = Files.createTempDirectory("ts-wrapper"); - // extract the ts-wrapper.zip to tempDir: - try (ZipInputStream zis = new ZipInputStream(new FileInputStream(tsWrapperZip))) { - ZipEntry entry = zis.getNextEntry(); - while (entry != null) { - Path entryPath = tempDir.resolve(entry.getName()); - if (entry.isDirectory()) { - Files.createDirectories(entryPath); - } else { - Files.copy(zis, entryPath); - } - entry = zis.getNextEntry(); - } - } + extractZip(tsWrapperZip, tempDir); + extractZip(tsNativeZip, tempDir.resolve("typescript-parser-wrapper/node_modules/@typescript")); Path tsWrapper = tempDir.resolve("typescript-parser-wrapper/main.js"); if (!Files.exists(tsWrapper)) { throw new RuntimeException("Could not find ts-wrapper at " + tsWrapper); @@ -60,4 +51,31 @@ public static void setUp() throws Exception { Env.systemEnv().pushEnvironmentContext(envUpdate); } + + private static void extractZip(String filename, Path destination) throws IOException { + // Unix modes are in the central directory, which ZipInputStream does not expose. + try (ZipFile zip = new ZipFile(Path.of(filename).toFile())) { + var entries = zip.getEntries(); + while (entries.hasMoreElements()) { + ZipArchiveEntry entry = entries.nextElement(); + Path entryPath = destination.resolve(entry.getName()).normalize(); + if (!entryPath.startsWith(destination)) { + throw new IOException("ZIP entry is outside its destination: " + entry.getName()); + } + if (entry.isDirectory()) { + Files.createDirectories(entryPath); + } else { + Files.createDirectories(entryPath.getParent()); + try (InputStream input = zip.getInputStream(entry)) { + Files.copy(input, entryPath); + } + if ((entry.getUnixMode() & 0111) != 0 + && entryPath.getFileSystem().supportedFileAttributeViews().contains("posix") + && !entryPath.toFile().setExecutable(true, false)) { + throw new IOException("Could not make ZIP entry executable: " + entryPath); + } + } + } + } + } } diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel b/javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel index 2d6a5d01c660..150827d9071f 100644 --- a/javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel +++ b/javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel @@ -5,15 +5,21 @@ java_test( srcs = glob(["**/*.java"]), data = [ "//javascript/extractor/lib/typescript", + "//javascript/extractor/lib/typescript:native", "//javascript/extractor/parser-tests", "//javascript/extractor/tests", "@nodejs//:node_bin", ], env = { "NODE_BIN": "$(rlocationpath @nodejs//:node_bin)", + "TS_NATIVE_ZIP": "$(rlocationpath //javascript/extractor/lib/typescript:native)", "TS_WRAPPER_ZIP": "$(rlocationpath //javascript/extractor/lib/typescript)", }, test_class = "com.semmle.js.extractor.test.AllTests", + runtime_deps = [ + "@semmle_code//resources/lib/java:commons-io", + "@semmle_code//resources/lib/java:commons-lang3", + ], # To use `replaceExpectedOutput` you need to uncomment the following line # (to be allowed to override the .trap files on disk) # tags = ["no-sandbox"], diff --git a/javascript/extractor/tests/ts/output/trap/import-attributes.ts.trap b/javascript/extractor/tests/ts/output/trap/import-attributes.ts.trap index 193a2afeb185..91161d974584 100644 --- a/javascript/extractor/tests/ts/output/trap/import-attributes.ts.trap +++ b/javascript/extractor/tests/ts/output/trap/import-attributes.ts.trap @@ -1311,837 +1311,1089 @@ properties(#20503,#20501,0,0,"type: ""json""") locations_default(#20504,#10000,1,24,1,35) hasLocation(#20503,#20504) #20505=* -stmts(#20505,27,#20001,1,"import ... son"" };") -hasLocation(#20505,#20009) -stmt_containers(#20505,#20001) +exprs(#20505,0,#20503,0,"type") +hasLocation(#20505,#20063) +enclosing_stmt(#20505,#20497) +expr_containers(#20505,#20001) +literals("type","type",#20505) #20506=* -exprs(#20506,4,#20505,-1,"""module""") -hasLocation(#20506,#20083) -enclosing_stmt(#20506,#20505) +exprs(#20506,4,#20503,1,"""json""") +hasLocation(#20506,#20067) +enclosing_stmt(#20506,#20497) expr_containers(#20506,#20001) -literals("module","""module""",#20506) +literals("json","""json""",#20506) #20507=* -regexpterm(#20507,14,#20506,0,"module") -#20508=@"loc,{#10000},2,22,2,27" -locations_default(#20508,#10000,2,22,2,27) +regexpterm(#20507,14,#20506,0,"json") +#20508=@"loc,{#10000},1,31,1,34" +locations_default(#20508,#10000,1,31,1,34) hasLocation(#20507,#20508) -regexp_const_value(#20507,"module") +regexp_const_value(#20507,"json") #20509=* -exprs(#20509,8,#20505,-10,"{ type: ""json"" }") -#20510=@"loc,{#10000},2,35,2,50" -locations_default(#20510,#10000,2,35,2,50) -hasLocation(#20509,#20510) -enclosing_stmt(#20509,#20505) -expr_containers(#20509,#20001) +stmts(#20509,27,#20001,1,"import ... son"" };") +hasLocation(#20509,#20009) +stmt_containers(#20509,#20001) +#20510=* +exprs(#20510,4,#20509,-1,"""module""") +hasLocation(#20510,#20083) +enclosing_stmt(#20510,#20509) +expr_containers(#20510,#20001) +literals("module","""module""",#20510) #20511=* -properties(#20511,#20509,0,0,"type: ""json""") -#20512=@"loc,{#10000},2,37,2,48" -locations_default(#20512,#10000,2,37,2,48) +regexpterm(#20511,14,#20510,0,"module") +#20512=@"loc,{#10000},2,22,2,27" +locations_default(#20512,#10000,2,22,2,27) hasLocation(#20511,#20512) +regexp_const_value(#20511,"module") #20513=* -exprs(#20513,85,#20505,0,"* as v1") -#20514=@"loc,{#10000},2,8,2,14" -locations_default(#20514,#10000,2,8,2,14) +exprs(#20513,8,#20509,-10,"{ type: ""json"" }") +#20514=@"loc,{#10000},2,35,2,50" +locations_default(#20514,#10000,2,35,2,50) hasLocation(#20513,#20514) -enclosing_stmt(#20513,#20505) +enclosing_stmt(#20513,#20509) expr_containers(#20513,#20001) #20515=* -exprs(#20515,78,#20513,1,"v1") -hasLocation(#20515,#20079) -enclosing_stmt(#20515,#20505) -expr_containers(#20515,#20001) -literals("v1","v1",#20515) -decl(#20515,#20486) -typedecl(#20515,#20489) -namespacedecl(#20515,#20492) -#20516=* -stmts(#20516,27,#20001,2,"import ... son"" };") -hasLocation(#20516,#20011) -stmt_containers(#20516,#20001) +properties(#20515,#20513,0,0,"type: ""json""") +#20516=@"loc,{#10000},2,37,2,48" +locations_default(#20516,#10000,2,37,2,48) +hasLocation(#20515,#20516) #20517=* -exprs(#20517,4,#20516,-1,"""module""") -hasLocation(#20517,#20109) -enclosing_stmt(#20517,#20516) +exprs(#20517,0,#20515,0,"type") +hasLocation(#20517,#20089) +enclosing_stmt(#20517,#20509) expr_containers(#20517,#20001) -literals("module","""module""",#20517) +literals("type","type",#20517) #20518=* -regexpterm(#20518,14,#20517,0,"module") -#20519=@"loc,{#10000},3,21,3,26" -locations_default(#20519,#10000,3,21,3,26) -hasLocation(#20518,#20519) -regexp_const_value(#20518,"module") -#20520=* -exprs(#20520,8,#20516,-10,"{ type: ""json"" }") -#20521=@"loc,{#10000},3,34,3,49" -locations_default(#20521,#10000,3,34,3,49) -hasLocation(#20520,#20521) -enclosing_stmt(#20520,#20516) -expr_containers(#20520,#20001) -#20522=* -properties(#20522,#20520,0,0,"type: ""json""") -#20523=@"loc,{#10000},3,36,3,47" -locations_default(#20523,#10000,3,36,3,47) -hasLocation(#20522,#20523) +exprs(#20518,4,#20515,1,"""json""") +hasLocation(#20518,#20093) +enclosing_stmt(#20518,#20509) +expr_containers(#20518,#20001) +literals("json","""json""",#20518) +#20519=* +regexpterm(#20519,14,#20518,0,"json") +#20520=@"loc,{#10000},2,44,2,47" +locations_default(#20520,#10000,2,44,2,47) +hasLocation(#20519,#20520) +regexp_const_value(#20519,"json") +#20521=* +exprs(#20521,85,#20509,0,"* as v1") +#20522=@"loc,{#10000},2,8,2,14" +locations_default(#20522,#10000,2,8,2,14) +hasLocation(#20521,#20522) +enclosing_stmt(#20521,#20509) +expr_containers(#20521,#20001) +#20523=* +exprs(#20523,78,#20521,1,"v1") +hasLocation(#20523,#20079) +enclosing_stmt(#20523,#20509) +expr_containers(#20523,#20001) +literals("v1","v1",#20523) +decl(#20523,#20486) +typedecl(#20523,#20489) +namespacedecl(#20523,#20492) #20524=* -exprs(#20524,83,#20516,0,"v2") -hasLocation(#20524,#20103) -enclosing_stmt(#20524,#20516) -expr_containers(#20524,#20001) +stmts(#20524,27,#20001,2,"import ... son"" };") +hasLocation(#20524,#20011) +stmt_containers(#20524,#20001) #20525=* -exprs(#20525,0,#20524,0,"v2") -hasLocation(#20525,#20103) -enclosing_stmt(#20525,#20516) +exprs(#20525,4,#20524,-1,"""module""") +hasLocation(#20525,#20109) +enclosing_stmt(#20525,#20524) expr_containers(#20525,#20001) -literals("v2","v2",#20525) +literals("module","""module""",#20525) #20526=* -exprs(#20526,78,#20524,1,"v2") -hasLocation(#20526,#20103) -enclosing_stmt(#20526,#20516) -expr_containers(#20526,#20001) -literals("v2","v2",#20526) -decl(#20526,#20487) -typedecl(#20526,#20490) -namespacedecl(#20526,#20493) -#20527=* -stmts(#20527,27,#20001,3,"import ... son"" };") -hasLocation(#20527,#20013) -stmt_containers(#20527,#20001) +regexpterm(#20526,14,#20525,0,"module") +#20527=@"loc,{#10000},3,21,3,26" +locations_default(#20527,#10000,3,21,3,26) +hasLocation(#20526,#20527) +regexp_const_value(#20526,"module") #20528=* -exprs(#20528,4,#20527,-1,"""module""") -hasLocation(#20528,#20131) -enclosing_stmt(#20528,#20527) +exprs(#20528,8,#20524,-10,"{ type: ""json"" }") +#20529=@"loc,{#10000},3,34,3,49" +locations_default(#20529,#10000,3,34,3,49) +hasLocation(#20528,#20529) +enclosing_stmt(#20528,#20524) expr_containers(#20528,#20001) -literals("module","""module""",#20528) -#20529=* -regexpterm(#20529,14,#20528,0,"module") -#20530=@"loc,{#10000},4,17,4,22" -locations_default(#20530,#10000,4,17,4,22) -hasLocation(#20529,#20530) -regexp_const_value(#20529,"module") -#20531=* -exprs(#20531,8,#20527,-10,"{ type: ""json"" }") -#20532=@"loc,{#10000},4,30,4,45" -locations_default(#20532,#10000,4,30,4,45) -hasLocation(#20531,#20532) -enclosing_stmt(#20531,#20527) -expr_containers(#20531,#20001) +#20530=* +properties(#20530,#20528,0,0,"type: ""json""") +#20531=@"loc,{#10000},3,36,3,47" +locations_default(#20531,#10000,3,36,3,47) +hasLocation(#20530,#20531) +#20532=* +exprs(#20532,0,#20530,0,"type") +hasLocation(#20532,#20115) +enclosing_stmt(#20532,#20524) +expr_containers(#20532,#20001) +literals("type","type",#20532) #20533=* -properties(#20533,#20531,0,0,"type: ""json""") -#20534=@"loc,{#10000},4,32,4,43" -locations_default(#20534,#10000,4,32,4,43) -hasLocation(#20533,#20534) -#20535=* -exprs(#20535,84,#20527,0,"v3") -hasLocation(#20535,#20127) -enclosing_stmt(#20535,#20527) -expr_containers(#20535,#20001) +exprs(#20533,4,#20530,1,"""json""") +hasLocation(#20533,#20119) +enclosing_stmt(#20533,#20524) +expr_containers(#20533,#20001) +literals("json","""json""",#20533) +#20534=* +regexpterm(#20534,14,#20533,0,"json") +#20535=@"loc,{#10000},3,43,3,46" +locations_default(#20535,#10000,3,43,3,46) +hasLocation(#20534,#20535) +regexp_const_value(#20534,"json") #20536=* -exprs(#20536,78,#20535,1,"v3") -hasLocation(#20536,#20127) -enclosing_stmt(#20536,#20527) +exprs(#20536,83,#20524,0,"v2") +hasLocation(#20536,#20103) +enclosing_stmt(#20536,#20524) expr_containers(#20536,#20001) -literals("v3","v3",#20536) -decl(#20536,#20488) -typedecl(#20536,#20491) -namespacedecl(#20536,#20494) #20537=* -stmts(#20537,30,#20001,4,"export ... son"" };") -hasLocation(#20537,#20017) -stmt_containers(#20537,#20001) +exprs(#20537,0,#20536,0,"v2") +hasLocation(#20537,#20103) +enclosing_stmt(#20537,#20524) +expr_containers(#20537,#20001) +literals("v2","v2",#20537) #20538=* -exprs(#20538,4,#20537,-2,"""module""") -hasLocation(#20538,#20157) -enclosing_stmt(#20538,#20537) +exprs(#20538,78,#20536,1,"v2") +hasLocation(#20538,#20103) +enclosing_stmt(#20538,#20524) expr_containers(#20538,#20001) -literals("module","""module""",#20538) +literals("v2","v2",#20538) +decl(#20538,#20487) +typedecl(#20538,#20490) +namespacedecl(#20538,#20493) #20539=* -regexpterm(#20539,14,#20538,0,"module") -#20540=@"loc,{#10000},6,21,6,26" -locations_default(#20540,#10000,6,21,6,26) -hasLocation(#20539,#20540) -regexp_const_value(#20539,"module") +stmts(#20539,27,#20001,3,"import ... son"" };") +hasLocation(#20539,#20013) +stmt_containers(#20539,#20001) +#20540=* +exprs(#20540,4,#20539,-1,"""module""") +hasLocation(#20540,#20131) +enclosing_stmt(#20540,#20539) +expr_containers(#20540,#20001) +literals("module","""module""",#20540) #20541=* -exprs(#20541,8,#20537,-10,"{ type: ""json"" }") -#20542=@"loc,{#10000},6,34,6,49" -locations_default(#20542,#10000,6,34,6,49) +regexpterm(#20541,14,#20540,0,"module") +#20542=@"loc,{#10000},4,17,4,22" +locations_default(#20542,#10000,4,17,4,22) hasLocation(#20541,#20542) -enclosing_stmt(#20541,#20537) -expr_containers(#20541,#20001) +regexp_const_value(#20541,"module") #20543=* -properties(#20543,#20541,0,0,"type: ""json""") -#20544=@"loc,{#10000},6,36,6,47" -locations_default(#20544,#10000,6,36,6,47) +exprs(#20543,8,#20539,-10,"{ type: ""json"" }") +#20544=@"loc,{#10000},4,30,4,45" +locations_default(#20544,#10000,4,30,4,45) hasLocation(#20543,#20544) +enclosing_stmt(#20543,#20539) +expr_containers(#20543,#20001) #20545=* -exprs(#20545,86,#20537,0,"v4") -hasLocation(#20545,#20151) -enclosing_stmt(#20545,#20537) -expr_containers(#20545,#20001) -#20546=* -exprs(#20546,0,#20545,0,"v4") -hasLocation(#20546,#20151) -enclosing_stmt(#20546,#20537) -expr_containers(#20546,#20001) -literals("v4","v4",#20546) +properties(#20545,#20543,0,0,"type: ""json""") +#20546=@"loc,{#10000},4,32,4,43" +locations_default(#20546,#10000,4,32,4,43) +hasLocation(#20545,#20546) #20547=* -exprs(#20547,0,#20545,1,"v4") -hasLocation(#20547,#20151) -enclosing_stmt(#20547,#20537) +exprs(#20547,0,#20545,0,"type") +hasLocation(#20547,#20137) +enclosing_stmt(#20547,#20539) expr_containers(#20547,#20001) -literals("v4","v4",#20547) +literals("type","type",#20547) #20548=* -stmts(#20548,28,#20001,5,"export ... son"" };") -hasLocation(#20548,#20019) -stmt_containers(#20548,#20001) +exprs(#20548,4,#20545,1,"""json""") +hasLocation(#20548,#20141) +enclosing_stmt(#20548,#20539) +expr_containers(#20548,#20001) +literals("json","""json""",#20548) #20549=* -exprs(#20549,4,#20548,0,"""module""") -hasLocation(#20549,#20179) -enclosing_stmt(#20549,#20548) -expr_containers(#20549,#20001) -literals("module","""module""",#20549) -#20550=* -regexpterm(#20550,14,#20549,0,"module") -#20551=@"loc,{#10000},7,16,7,21" -locations_default(#20551,#10000,7,16,7,21) -hasLocation(#20550,#20551) -regexp_const_value(#20550,"module") +regexpterm(#20549,14,#20548,0,"json") +#20550=@"loc,{#10000},4,39,4,42" +locations_default(#20550,#10000,4,39,4,42) +hasLocation(#20549,#20550) +regexp_const_value(#20549,"json") +#20551=* +exprs(#20551,84,#20539,0,"v3") +hasLocation(#20551,#20127) +enclosing_stmt(#20551,#20539) +expr_containers(#20551,#20001) #20552=* -exprs(#20552,8,#20548,-10,"{ type: ""json"" }") -#20553=@"loc,{#10000},7,29,7,44" -locations_default(#20553,#10000,7,29,7,44) -hasLocation(#20552,#20553) -enclosing_stmt(#20552,#20548) +exprs(#20552,78,#20551,1,"v3") +hasLocation(#20552,#20127) +enclosing_stmt(#20552,#20539) expr_containers(#20552,#20001) +literals("v3","v3",#20552) +decl(#20552,#20488) +typedecl(#20552,#20491) +namespacedecl(#20552,#20494) +#20553=* +stmts(#20553,30,#20001,4,"export ... son"" };") +hasLocation(#20553,#20017) +stmt_containers(#20553,#20001) #20554=* -properties(#20554,#20552,0,0,"type: ""json""") -#20555=@"loc,{#10000},7,31,7,42" -locations_default(#20555,#10000,7,31,7,42) -hasLocation(#20554,#20555) -#20556=* -stmts(#20556,30,#20001,6,"export ... son"" };") -hasLocation(#20556,#20021) -stmt_containers(#20556,#20001) +exprs(#20554,4,#20553,-2,"""module""") +hasLocation(#20554,#20157) +enclosing_stmt(#20554,#20553) +expr_containers(#20554,#20001) +literals("module","""module""",#20554) +#20555=* +regexpterm(#20555,14,#20554,0,"module") +#20556=@"loc,{#10000},6,21,6,26" +locations_default(#20556,#10000,6,21,6,26) +hasLocation(#20555,#20556) +regexp_const_value(#20555,"module") #20557=* -exprs(#20557,4,#20556,-2,"""module""") -hasLocation(#20557,#20205) -enclosing_stmt(#20557,#20556) +exprs(#20557,8,#20553,-10,"{ type: ""json"" }") +#20558=@"loc,{#10000},6,34,6,49" +locations_default(#20558,#10000,6,34,6,49) +hasLocation(#20557,#20558) +enclosing_stmt(#20557,#20553) expr_containers(#20557,#20001) -literals("module","""module""",#20557) -#20558=* -regexpterm(#20558,14,#20557,0,"module") -#20559=@"loc,{#10000},8,22,8,27" -locations_default(#20559,#10000,8,22,8,27) -hasLocation(#20558,#20559) -regexp_const_value(#20558,"module") -#20560=* -exprs(#20560,8,#20556,-10,"{ type: ""json"" }") -#20561=@"loc,{#10000},8,35,8,50" -locations_default(#20561,#10000,8,35,8,50) -hasLocation(#20560,#20561) -enclosing_stmt(#20560,#20556) -expr_containers(#20560,#20001) +#20559=* +properties(#20559,#20557,0,0,"type: ""json""") +#20560=@"loc,{#10000},6,36,6,47" +locations_default(#20560,#10000,6,36,6,47) +hasLocation(#20559,#20560) +#20561=* +exprs(#20561,0,#20559,0,"type") +hasLocation(#20561,#20163) +enclosing_stmt(#20561,#20553) +expr_containers(#20561,#20001) +literals("type","type",#20561) #20562=* -properties(#20562,#20560,0,0,"type: ""json""") -#20563=@"loc,{#10000},8,37,8,48" -locations_default(#20563,#10000,8,37,8,48) -hasLocation(#20562,#20563) -#20564=* -exprs(#20564,96,#20556,0,"* as v5") -#20565=@"loc,{#10000},8,8,8,14" -locations_default(#20565,#10000,8,8,8,14) -hasLocation(#20564,#20565) -enclosing_stmt(#20564,#20556) -expr_containers(#20564,#20001) +exprs(#20562,4,#20559,1,"""json""") +hasLocation(#20562,#20167) +enclosing_stmt(#20562,#20553) +expr_containers(#20562,#20001) +literals("json","""json""",#20562) +#20563=* +regexpterm(#20563,14,#20562,0,"json") +#20564=@"loc,{#10000},6,43,6,46" +locations_default(#20564,#10000,6,43,6,46) +hasLocation(#20563,#20564) +regexp_const_value(#20563,"json") +#20565=* +exprs(#20565,86,#20553,0,"v4") +hasLocation(#20565,#20151) +enclosing_stmt(#20565,#20553) +expr_containers(#20565,#20001) #20566=* -exprs(#20566,0,#20564,1,"v5") -hasLocation(#20566,#20201) -enclosing_stmt(#20566,#20556) +exprs(#20566,0,#20565,0,"v4") +hasLocation(#20566,#20151) +enclosing_stmt(#20566,#20553) expr_containers(#20566,#20001) -literals("v5","v5",#20566) +literals("v4","v4",#20566) #20567=* -stmts(#20567,22,#20001,7,"const v ... "" } });") -hasLocation(#20567,#20025) -stmt_containers(#20567,#20001) +exprs(#20567,0,#20565,1,"v4") +hasLocation(#20567,#20151) +enclosing_stmt(#20567,#20553) +expr_containers(#20567,#20001) +literals("v4","v4",#20567) #20568=* -exprs(#20568,64,#20567,0,"v6 = im ... n"" } })") -#20569=@"loc,{#10000},10,7,10,57" -locations_default(#20569,#10000,10,7,10,57) -hasLocation(#20568,#20569) -enclosing_stmt(#20568,#20567) -expr_containers(#20568,#20001) +stmts(#20568,28,#20001,5,"export ... son"" };") +hasLocation(#20568,#20019) +stmt_containers(#20568,#20001) +#20569=* +exprs(#20569,4,#20568,0,"""module""") +hasLocation(#20569,#20179) +enclosing_stmt(#20569,#20568) +expr_containers(#20569,#20001) +literals("module","""module""",#20569) #20570=* -exprs(#20570,78,#20568,0,"v6") -hasLocation(#20570,#20223) -enclosing_stmt(#20570,#20567) -expr_containers(#20570,#20001) -literals("v6","v6",#20570) -decl(#20570,#20495) -#20571=* -exprs(#20571,99,#20568,1,"import( ... n"" } })") -#20572=@"loc,{#10000},10,12,10,57" -locations_default(#20572,#10000,10,12,10,57) -hasLocation(#20571,#20572) -enclosing_stmt(#20571,#20567) -expr_containers(#20571,#20001) -#20573=* -exprs(#20573,4,#20571,0,"""module""") -hasLocation(#20573,#20231) -enclosing_stmt(#20573,#20567) -expr_containers(#20573,#20001) -literals("module","""module""",#20573) +regexpterm(#20570,14,#20569,0,"module") +#20571=@"loc,{#10000},7,16,7,21" +locations_default(#20571,#10000,7,16,7,21) +hasLocation(#20570,#20571) +regexp_const_value(#20570,"module") +#20572=* +exprs(#20572,8,#20568,-10,"{ type: ""json"" }") +#20573=@"loc,{#10000},7,29,7,44" +locations_default(#20573,#10000,7,29,7,44) +hasLocation(#20572,#20573) +enclosing_stmt(#20572,#20568) +expr_containers(#20572,#20001) #20574=* -regexpterm(#20574,14,#20573,0,"module") -#20575=@"loc,{#10000},10,20,10,25" -locations_default(#20575,#10000,10,20,10,25) +properties(#20574,#20572,0,0,"type: ""json""") +#20575=@"loc,{#10000},7,31,7,42" +locations_default(#20575,#10000,7,31,7,42) hasLocation(#20574,#20575) -regexp_const_value(#20574,"module") #20576=* -exprs(#20576,8,#20571,1,"{ ""with ... on"" } }") -#20577=@"loc,{#10000},10,29,10,56" -locations_default(#20577,#10000,10,29,10,56) -hasLocation(#20576,#20577) -enclosing_stmt(#20576,#20567) +exprs(#20576,0,#20574,0,"type") +hasLocation(#20576,#20185) +enclosing_stmt(#20576,#20568) expr_containers(#20576,#20001) +literals("type","type",#20576) +#20577=* +exprs(#20577,4,#20574,1,"""json""") +hasLocation(#20577,#20189) +enclosing_stmt(#20577,#20568) +expr_containers(#20577,#20001) +literals("json","""json""",#20577) #20578=* -properties(#20578,#20576,0,0,"""with"": ... json"" }") -#20579=@"loc,{#10000},10,31,10,54" -locations_default(#20579,#10000,10,31,10,54) +regexpterm(#20578,14,#20577,0,"json") +#20579=@"loc,{#10000},7,38,7,41" +locations_default(#20579,#10000,7,38,7,41) hasLocation(#20578,#20579) +regexp_const_value(#20578,"json") #20580=* -exprs(#20580,4,#20578,0,"""with""") -hasLocation(#20580,#20237) -enclosing_stmt(#20580,#20567) -expr_containers(#20580,#20001) -literals("with","""with""",#20580) +stmts(#20580,30,#20001,6,"export ... son"" };") +hasLocation(#20580,#20021) +stmt_containers(#20580,#20001) #20581=* -regexpterm(#20581,14,#20580,0,"with") -#20582=@"loc,{#10000},10,32,10,35" -locations_default(#20582,#10000,10,32,10,35) -hasLocation(#20581,#20582) -regexp_const_value(#20581,"with") -#20583=* -exprs(#20583,8,#20578,1,"{ type: ""json"" }") -#20584=@"loc,{#10000},10,39,10,54" -locations_default(#20584,#10000,10,39,10,54) -hasLocation(#20583,#20584) -enclosing_stmt(#20583,#20567) -expr_containers(#20583,#20001) -#20585=* -properties(#20585,#20583,0,0,"type: ""json""") -#20586=@"loc,{#10000},10,41,10,52" -locations_default(#20586,#10000,10,41,10,52) -hasLocation(#20585,#20586) -#20587=* -exprs(#20587,0,#20585,0,"type") -hasLocation(#20587,#20243) -enclosing_stmt(#20587,#20567) -expr_containers(#20587,#20001) -literals("type","type",#20587) +exprs(#20581,4,#20580,-2,"""module""") +hasLocation(#20581,#20205) +enclosing_stmt(#20581,#20580) +expr_containers(#20581,#20001) +literals("module","""module""",#20581) +#20582=* +regexpterm(#20582,14,#20581,0,"module") +#20583=@"loc,{#10000},8,22,8,27" +locations_default(#20583,#10000,8,22,8,27) +hasLocation(#20582,#20583) +regexp_const_value(#20582,"module") +#20584=* +exprs(#20584,8,#20580,-10,"{ type: ""json"" }") +#20585=@"loc,{#10000},8,35,8,50" +locations_default(#20585,#10000,8,35,8,50) +hasLocation(#20584,#20585) +enclosing_stmt(#20584,#20580) +expr_containers(#20584,#20001) +#20586=* +properties(#20586,#20584,0,0,"type: ""json""") +#20587=@"loc,{#10000},8,37,8,48" +locations_default(#20587,#10000,8,37,8,48) +hasLocation(#20586,#20587) #20588=* -exprs(#20588,4,#20585,1,"""json""") -hasLocation(#20588,#20247) -enclosing_stmt(#20588,#20567) +exprs(#20588,0,#20586,0,"type") +hasLocation(#20588,#20211) +enclosing_stmt(#20588,#20580) expr_containers(#20588,#20001) -literals("json","""json""",#20588) +literals("type","type",#20588) #20589=* -regexpterm(#20589,14,#20588,0,"json") -#20590=@"loc,{#10000},10,48,10,51" -locations_default(#20590,#10000,10,48,10,51) -hasLocation(#20589,#20590) -regexp_const_value(#20589,"json") -#20591=* -stmts(#20591,27,#20001,8,"import ""module"";") -#20592=@"loc,{#10000},12,1,12,16" -locations_default(#20592,#10000,12,1,12,16) -hasLocation(#20591,#20592) -stmt_containers(#20591,#20001) -#20593=* -exprs(#20593,4,#20591,-1,"""module""") -hasLocation(#20593,#20259) -enclosing_stmt(#20593,#20591) -expr_containers(#20593,#20001) -literals("module","""module""",#20593) +exprs(#20589,4,#20586,1,"""json""") +hasLocation(#20589,#20215) +enclosing_stmt(#20589,#20580) +expr_containers(#20589,#20001) +literals("json","""json""",#20589) +#20590=* +regexpterm(#20590,14,#20589,0,"json") +#20591=@"loc,{#10000},8,44,8,47" +locations_default(#20591,#10000,8,44,8,47) +hasLocation(#20590,#20591) +regexp_const_value(#20590,"json") +#20592=* +exprs(#20592,96,#20580,0,"* as v5") +#20593=@"loc,{#10000},8,8,8,14" +locations_default(#20593,#10000,8,8,8,14) +hasLocation(#20592,#20593) +enclosing_stmt(#20592,#20580) +expr_containers(#20592,#20001) #20594=* -regexpterm(#20594,14,#20593,0,"module") -#20595=@"loc,{#10000},12,9,12,14" -locations_default(#20595,#10000,12,9,12,14) -hasLocation(#20594,#20595) -regexp_const_value(#20594,"module") +exprs(#20594,0,#20592,1,"v5") +hasLocation(#20594,#20201) +enclosing_stmt(#20594,#20580) +expr_containers(#20594,#20001) +literals("v5","v5",#20594) +#20595=* +stmts(#20595,22,#20001,7,"const v ... "" } });") +hasLocation(#20595,#20025) +stmt_containers(#20595,#20001) #20596=* -stmts(#20596,2,#20001,9,"assert( ... on"" });") -#20597=@"loc,{#10000},13,1,13,25" -locations_default(#20597,#10000,13,1,13,25) +exprs(#20596,64,#20595,0,"v6 = im ... n"" } })") +#20597=@"loc,{#10000},10,7,10,57" +locations_default(#20597,#10000,10,7,10,57) hasLocation(#20596,#20597) -stmt_containers(#20596,#20001) +enclosing_stmt(#20596,#20595) +expr_containers(#20596,#20001) #20598=* -exprs(#20598,13,#20596,0,"assert( ... son"" })") -#20599=@"loc,{#10000},13,1,13,24" -locations_default(#20599,#10000,13,1,13,24) -hasLocation(#20598,#20599) -enclosing_stmt(#20598,#20596) +exprs(#20598,78,#20596,0,"v6") +hasLocation(#20598,#20223) +enclosing_stmt(#20598,#20595) expr_containers(#20598,#20001) -#20600=* -exprs(#20600,79,#20598,-1,"assert") -hasLocation(#20600,#20263) -enclosing_stmt(#20600,#20596) -expr_containers(#20600,#20001) -literals("assert","assert",#20600) -#20601=@"var;{assert};{#20000}" -variables(#20601,"assert",#20000) -bind(#20600,#20601) +literals("v6","v6",#20598) +decl(#20598,#20495) +#20599=* +exprs(#20599,99,#20596,1,"import( ... n"" } })") +#20600=@"loc,{#10000},10,12,10,57" +locations_default(#20600,#10000,10,12,10,57) +hasLocation(#20599,#20600) +enclosing_stmt(#20599,#20595) +expr_containers(#20599,#20001) +#20601=* +exprs(#20601,4,#20599,0,"""module""") +hasLocation(#20601,#20231) +enclosing_stmt(#20601,#20595) +expr_containers(#20601,#20001) +literals("module","""module""",#20601) #20602=* -exprs(#20602,8,#20598,0,"{ type: ""json"" }") -#20603=@"loc,{#10000},13,8,13,23" -locations_default(#20603,#10000,13,8,13,23) +regexpterm(#20602,14,#20601,0,"module") +#20603=@"loc,{#10000},10,20,10,25" +locations_default(#20603,#10000,10,20,10,25) hasLocation(#20602,#20603) -enclosing_stmt(#20602,#20596) -expr_containers(#20602,#20001) +regexp_const_value(#20602,"module") #20604=* -properties(#20604,#20602,0,0,"type: ""json""") -#20605=@"loc,{#10000},13,10,13,21" -locations_default(#20605,#10000,13,10,13,21) +exprs(#20604,8,#20599,1,"{ ""with ... on"" } }") +#20605=@"loc,{#10000},10,29,10,56" +locations_default(#20605,#10000,10,29,10,56) hasLocation(#20604,#20605) +enclosing_stmt(#20604,#20595) +expr_containers(#20604,#20001) #20606=* -exprs(#20606,0,#20604,0,"type") -hasLocation(#20606,#20269) -enclosing_stmt(#20606,#20596) -expr_containers(#20606,#20001) -literals("type","type",#20606) -#20607=* -exprs(#20607,4,#20604,1,"""json""") -hasLocation(#20607,#20273) -enclosing_stmt(#20607,#20596) -expr_containers(#20607,#20001) -literals("json","""json""",#20607) +properties(#20606,#20604,0,0,"""with"": ... json"" }") +#20607=@"loc,{#10000},10,31,10,54" +locations_default(#20607,#10000,10,31,10,54) +hasLocation(#20606,#20607) #20608=* -regexpterm(#20608,14,#20607,0,"json") -#20609=@"loc,{#10000},13,17,13,20" -locations_default(#20609,#10000,13,17,13,20) -hasLocation(#20608,#20609) -regexp_const_value(#20608,"json") -#20610=* -stmts(#20610,27,#20001,10,"import ... son"" };") -hasLocation(#20610,#20035) -stmt_containers(#20610,#20001) +exprs(#20608,4,#20606,0,"""with""") +hasLocation(#20608,#20237) +enclosing_stmt(#20608,#20595) +expr_containers(#20608,#20001) +literals("with","""with""",#20608) +#20609=* +regexpterm(#20609,14,#20608,0,"with") +#20610=@"loc,{#10000},10,32,10,35" +locations_default(#20610,#10000,10,32,10,35) +hasLocation(#20609,#20610) +regexp_const_value(#20609,"with") #20611=* -exprs(#20611,4,#20610,-1,"""module""") -hasLocation(#20611,#20283) -enclosing_stmt(#20611,#20610) +exprs(#20611,8,#20606,1,"{ type: ""json"" }") +#20612=@"loc,{#10000},10,39,10,54" +locations_default(#20612,#10000,10,39,10,54) +hasLocation(#20611,#20612) +enclosing_stmt(#20611,#20595) expr_containers(#20611,#20001) -literals("module","""module""",#20611) -#20612=* -regexpterm(#20612,14,#20611,0,"module") -#20613=@"loc,{#10000},15,9,15,14" -locations_default(#20613,#10000,15,9,15,14) -hasLocation(#20612,#20613) -regexp_const_value(#20612,"module") -#20614=* -exprs(#20614,8,#20610,-10,"{ type: ""json"" }") -#20615=@"loc,{#10000},15,24,15,39" -locations_default(#20615,#10000,15,24,15,39) -hasLocation(#20614,#20615) -enclosing_stmt(#20614,#20610) -expr_containers(#20614,#20001) +#20613=* +properties(#20613,#20611,0,0,"type: ""json""") +#20614=@"loc,{#10000},10,41,10,52" +locations_default(#20614,#10000,10,41,10,52) +hasLocation(#20613,#20614) +#20615=* +exprs(#20615,0,#20613,0,"type") +hasLocation(#20615,#20243) +enclosing_stmt(#20615,#20595) +expr_containers(#20615,#20001) +literals("type","type",#20615) #20616=* -properties(#20616,#20614,0,0,"type: ""json""") -#20617=@"loc,{#10000},15,26,15,37" -locations_default(#20617,#10000,15,26,15,37) -hasLocation(#20616,#20617) -#20618=* -stmts(#20618,27,#20001,11,"import ... son"" };") -hasLocation(#20618,#20037) -stmt_containers(#20618,#20001) +exprs(#20616,4,#20613,1,"""json""") +hasLocation(#20616,#20247) +enclosing_stmt(#20616,#20595) +expr_containers(#20616,#20001) +literals("json","""json""",#20616) +#20617=* +regexpterm(#20617,14,#20616,0,"json") +#20618=@"loc,{#10000},10,48,10,51" +locations_default(#20618,#10000,10,48,10,51) +hasLocation(#20617,#20618) +regexp_const_value(#20617,"json") #20619=* -exprs(#20619,4,#20618,-1,"""module""") -hasLocation(#20619,#20309) -enclosing_stmt(#20619,#20618) -expr_containers(#20619,#20001) -literals("module","""module""",#20619) -#20620=* -regexpterm(#20620,14,#20619,0,"module") -#20621=@"loc,{#10000},16,22,16,27" -locations_default(#20621,#10000,16,22,16,27) -hasLocation(#20620,#20621) -regexp_const_value(#20620,"module") +stmts(#20619,27,#20001,8,"import ""module"";") +#20620=@"loc,{#10000},12,1,12,16" +locations_default(#20620,#10000,12,1,12,16) +hasLocation(#20619,#20620) +stmt_containers(#20619,#20001) +#20621=* +exprs(#20621,4,#20619,-1,"""module""") +hasLocation(#20621,#20259) +enclosing_stmt(#20621,#20619) +expr_containers(#20621,#20001) +literals("module","""module""",#20621) #20622=* -exprs(#20622,8,#20618,-10,"{ type: ""json"" }") -#20623=@"loc,{#10000},16,37,16,52" -locations_default(#20623,#10000,16,37,16,52) +regexpterm(#20622,14,#20621,0,"module") +#20623=@"loc,{#10000},12,9,12,14" +locations_default(#20623,#10000,12,9,12,14) hasLocation(#20622,#20623) -enclosing_stmt(#20622,#20618) -expr_containers(#20622,#20001) +regexp_const_value(#20622,"module") #20624=* -properties(#20624,#20622,0,0,"type: ""json""") -#20625=@"loc,{#10000},16,39,16,50" -locations_default(#20625,#10000,16,39,16,50) +stmts(#20624,2,#20001,9,"assert( ... on"" });") +#20625=@"loc,{#10000},13,1,13,25" +locations_default(#20625,#10000,13,1,13,25) hasLocation(#20624,#20625) +stmt_containers(#20624,#20001) #20626=* -exprs(#20626,85,#20618,0,"* as v1") -#20627=@"loc,{#10000},16,8,16,14" -locations_default(#20627,#10000,16,8,16,14) +exprs(#20626,13,#20624,0,"assert( ... son"" })") +#20627=@"loc,{#10000},13,1,13,24" +locations_default(#20627,#10000,13,1,13,24) hasLocation(#20626,#20627) -enclosing_stmt(#20626,#20618) +enclosing_stmt(#20626,#20624) expr_containers(#20626,#20001) #20628=* -exprs(#20628,78,#20626,1,"v1") -hasLocation(#20628,#20305) -enclosing_stmt(#20628,#20618) +exprs(#20628,79,#20626,-1,"assert") +hasLocation(#20628,#20263) +enclosing_stmt(#20628,#20624) expr_containers(#20628,#20001) -literals("v1","v1",#20628) -decl(#20628,#20486) -typedecl(#20628,#20489) -namespacedecl(#20628,#20492) -#20629=* -stmts(#20629,27,#20001,12,"import ... son"" };") -hasLocation(#20629,#20039) -stmt_containers(#20629,#20001) +literals("assert","assert",#20628) +#20629=@"var;{assert};{#20000}" +variables(#20629,"assert",#20000) +bind(#20628,#20629) #20630=* -exprs(#20630,4,#20629,-1,"""module""") -hasLocation(#20630,#20335) -enclosing_stmt(#20630,#20629) +exprs(#20630,8,#20626,0,"{ type: ""json"" }") +#20631=@"loc,{#10000},13,8,13,23" +locations_default(#20631,#10000,13,8,13,23) +hasLocation(#20630,#20631) +enclosing_stmt(#20630,#20624) expr_containers(#20630,#20001) -literals("module","""module""",#20630) -#20631=* -regexpterm(#20631,14,#20630,0,"module") -#20632=@"loc,{#10000},17,21,17,26" -locations_default(#20632,#10000,17,21,17,26) -hasLocation(#20631,#20632) -regexp_const_value(#20631,"module") -#20633=* -exprs(#20633,8,#20629,-10,"{ type: ""json"" }") -#20634=@"loc,{#10000},17,36,17,51" -locations_default(#20634,#10000,17,36,17,51) -hasLocation(#20633,#20634) -enclosing_stmt(#20633,#20629) -expr_containers(#20633,#20001) +#20632=* +properties(#20632,#20630,0,0,"type: ""json""") +#20633=@"loc,{#10000},13,10,13,21" +locations_default(#20633,#10000,13,10,13,21) +hasLocation(#20632,#20633) +#20634=* +exprs(#20634,0,#20632,0,"type") +hasLocation(#20634,#20269) +enclosing_stmt(#20634,#20624) +expr_containers(#20634,#20001) +literals("type","type",#20634) #20635=* -properties(#20635,#20633,0,0,"type: ""json""") -#20636=@"loc,{#10000},17,38,17,49" -locations_default(#20636,#10000,17,38,17,49) -hasLocation(#20635,#20636) -#20637=* -exprs(#20637,83,#20629,0,"v2") -hasLocation(#20637,#20329) -enclosing_stmt(#20637,#20629) -expr_containers(#20637,#20001) +exprs(#20635,4,#20632,1,"""json""") +hasLocation(#20635,#20273) +enclosing_stmt(#20635,#20624) +expr_containers(#20635,#20001) +literals("json","""json""",#20635) +#20636=* +regexpterm(#20636,14,#20635,0,"json") +#20637=@"loc,{#10000},13,17,13,20" +locations_default(#20637,#10000,13,17,13,20) +hasLocation(#20636,#20637) +regexp_const_value(#20636,"json") #20638=* -exprs(#20638,0,#20637,0,"v2") -hasLocation(#20638,#20329) -enclosing_stmt(#20638,#20629) -expr_containers(#20638,#20001) -literals("v2","v2",#20638) +stmts(#20638,27,#20001,10,"import ... son"" };") +hasLocation(#20638,#20035) +stmt_containers(#20638,#20001) #20639=* -exprs(#20639,78,#20637,1,"v2") -hasLocation(#20639,#20329) -enclosing_stmt(#20639,#20629) +exprs(#20639,4,#20638,-1,"""module""") +hasLocation(#20639,#20283) +enclosing_stmt(#20639,#20638) expr_containers(#20639,#20001) -literals("v2","v2",#20639) -decl(#20639,#20487) -typedecl(#20639,#20490) -namespacedecl(#20639,#20493) +literals("module","""module""",#20639) #20640=* -stmts(#20640,27,#20001,13,"import ... son"" };") -hasLocation(#20640,#20041) -stmt_containers(#20640,#20001) -#20641=* -exprs(#20641,4,#20640,-1,"""module""") -hasLocation(#20641,#20357) -enclosing_stmt(#20641,#20640) -expr_containers(#20641,#20001) -literals("module","""module""",#20641) +regexpterm(#20640,14,#20639,0,"module") +#20641=@"loc,{#10000},15,9,15,14" +locations_default(#20641,#10000,15,9,15,14) +hasLocation(#20640,#20641) +regexp_const_value(#20640,"module") #20642=* -regexpterm(#20642,14,#20641,0,"module") -#20643=@"loc,{#10000},18,17,18,22" -locations_default(#20643,#10000,18,17,18,22) +exprs(#20642,8,#20638,-10,"{ type: ""json"" }") +#20643=@"loc,{#10000},15,24,15,39" +locations_default(#20643,#10000,15,24,15,39) hasLocation(#20642,#20643) -regexp_const_value(#20642,"module") +enclosing_stmt(#20642,#20638) +expr_containers(#20642,#20001) #20644=* -exprs(#20644,8,#20640,-10,"{ type: ""json"" }") -#20645=@"loc,{#10000},18,32,18,47" -locations_default(#20645,#10000,18,32,18,47) +properties(#20644,#20642,0,0,"type: ""json""") +#20645=@"loc,{#10000},15,26,15,37" +locations_default(#20645,#10000,15,26,15,37) hasLocation(#20644,#20645) -enclosing_stmt(#20644,#20640) -expr_containers(#20644,#20001) #20646=* -properties(#20646,#20644,0,0,"type: ""json""") -#20647=@"loc,{#10000},18,34,18,45" -locations_default(#20647,#10000,18,34,18,45) -hasLocation(#20646,#20647) +exprs(#20646,0,#20644,0,"type") +hasLocation(#20646,#20289) +enclosing_stmt(#20646,#20638) +expr_containers(#20646,#20001) +literals("type","type",#20646) +#20647=* +exprs(#20647,4,#20644,1,"""json""") +hasLocation(#20647,#20293) +enclosing_stmt(#20647,#20638) +expr_containers(#20647,#20001) +literals("json","""json""",#20647) #20648=* -exprs(#20648,84,#20640,0,"v3") -hasLocation(#20648,#20353) -enclosing_stmt(#20648,#20640) -expr_containers(#20648,#20001) -#20649=* -exprs(#20649,78,#20648,1,"v3") -hasLocation(#20649,#20353) -enclosing_stmt(#20649,#20640) -expr_containers(#20649,#20001) -literals("v3","v3",#20649) -decl(#20649,#20488) -typedecl(#20649,#20491) -namespacedecl(#20649,#20494) +regexpterm(#20648,14,#20647,0,"json") +#20649=@"loc,{#10000},15,33,15,36" +locations_default(#20649,#10000,15,33,15,36) +hasLocation(#20648,#20649) +regexp_const_value(#20648,"json") #20650=* -stmts(#20650,30,#20001,14,"export ... son"" };") -hasLocation(#20650,#20045) +stmts(#20650,27,#20001,11,"import ... son"" };") +hasLocation(#20650,#20037) stmt_containers(#20650,#20001) #20651=* -exprs(#20651,4,#20650,-2,"""module""") -hasLocation(#20651,#20383) +exprs(#20651,4,#20650,-1,"""module""") +hasLocation(#20651,#20309) enclosing_stmt(#20651,#20650) expr_containers(#20651,#20001) literals("module","""module""",#20651) #20652=* regexpterm(#20652,14,#20651,0,"module") -#20653=@"loc,{#10000},20,21,20,26" -locations_default(#20653,#10000,20,21,20,26) +#20653=@"loc,{#10000},16,22,16,27" +locations_default(#20653,#10000,16,22,16,27) hasLocation(#20652,#20653) regexp_const_value(#20652,"module") #20654=* exprs(#20654,8,#20650,-10,"{ type: ""json"" }") -#20655=@"loc,{#10000},20,36,20,51" -locations_default(#20655,#10000,20,36,20,51) +#20655=@"loc,{#10000},16,37,16,52" +locations_default(#20655,#10000,16,37,16,52) hasLocation(#20654,#20655) enclosing_stmt(#20654,#20650) expr_containers(#20654,#20001) #20656=* properties(#20656,#20654,0,0,"type: ""json""") -#20657=@"loc,{#10000},20,38,20,49" -locations_default(#20657,#10000,20,38,20,49) +#20657=@"loc,{#10000},16,39,16,50" +locations_default(#20657,#10000,16,39,16,50) hasLocation(#20656,#20657) #20658=* -exprs(#20658,86,#20650,0,"v4") -hasLocation(#20658,#20377) +exprs(#20658,0,#20656,0,"type") +hasLocation(#20658,#20315) enclosing_stmt(#20658,#20650) expr_containers(#20658,#20001) +literals("type","type",#20658) #20659=* -exprs(#20659,0,#20658,0,"v4") -hasLocation(#20659,#20377) +exprs(#20659,4,#20656,1,"""json""") +hasLocation(#20659,#20319) enclosing_stmt(#20659,#20650) expr_containers(#20659,#20001) -literals("v4","v4",#20659) +literals("json","""json""",#20659) #20660=* -exprs(#20660,0,#20658,1,"v4") -hasLocation(#20660,#20377) -enclosing_stmt(#20660,#20650) -expr_containers(#20660,#20001) -literals("v4","v4",#20660) -#20661=* -stmts(#20661,28,#20001,15,"export ... son"" };") -hasLocation(#20661,#20047) -stmt_containers(#20661,#20001) +regexpterm(#20660,14,#20659,0,"json") +#20661=@"loc,{#10000},16,46,16,49" +locations_default(#20661,#10000,16,46,16,49) +hasLocation(#20660,#20661) +regexp_const_value(#20660,"json") #20662=* -exprs(#20662,4,#20661,0,"""module""") -hasLocation(#20662,#20405) -enclosing_stmt(#20662,#20661) +exprs(#20662,85,#20650,0,"* as v1") +#20663=@"loc,{#10000},16,8,16,14" +locations_default(#20663,#10000,16,8,16,14) +hasLocation(#20662,#20663) +enclosing_stmt(#20662,#20650) expr_containers(#20662,#20001) -literals("module","""module""",#20662) -#20663=* -regexpterm(#20663,14,#20662,0,"module") -#20664=@"loc,{#10000},21,16,21,21" -locations_default(#20664,#10000,21,16,21,21) -hasLocation(#20663,#20664) -regexp_const_value(#20663,"module") +#20664=* +exprs(#20664,78,#20662,1,"v1") +hasLocation(#20664,#20305) +enclosing_stmt(#20664,#20650) +expr_containers(#20664,#20001) +literals("v1","v1",#20664) +decl(#20664,#20486) +typedecl(#20664,#20489) +namespacedecl(#20664,#20492) #20665=* -exprs(#20665,8,#20661,-10,"{ type: ""json"" }") -#20666=@"loc,{#10000},21,31,21,46" -locations_default(#20666,#10000,21,31,21,46) -hasLocation(#20665,#20666) -enclosing_stmt(#20665,#20661) -expr_containers(#20665,#20001) +stmts(#20665,27,#20001,12,"import ... son"" };") +hasLocation(#20665,#20039) +stmt_containers(#20665,#20001) +#20666=* +exprs(#20666,4,#20665,-1,"""module""") +hasLocation(#20666,#20335) +enclosing_stmt(#20666,#20665) +expr_containers(#20666,#20001) +literals("module","""module""",#20666) #20667=* -properties(#20667,#20665,0,0,"type: ""json""") -#20668=@"loc,{#10000},21,33,21,44" -locations_default(#20668,#10000,21,33,21,44) +regexpterm(#20667,14,#20666,0,"module") +#20668=@"loc,{#10000},17,21,17,26" +locations_default(#20668,#10000,17,21,17,26) hasLocation(#20667,#20668) +regexp_const_value(#20667,"module") #20669=* -stmts(#20669,30,#20001,16,"export ... son"" };") -hasLocation(#20669,#20049) -stmt_containers(#20669,#20001) -#20670=* -exprs(#20670,4,#20669,-2,"""module""") -hasLocation(#20670,#20431) -enclosing_stmt(#20670,#20669) -expr_containers(#20670,#20001) -literals("module","""module""",#20670) +exprs(#20669,8,#20665,-10,"{ type: ""json"" }") +#20670=@"loc,{#10000},17,36,17,51" +locations_default(#20670,#10000,17,36,17,51) +hasLocation(#20669,#20670) +enclosing_stmt(#20669,#20665) +expr_containers(#20669,#20001) #20671=* -regexpterm(#20671,14,#20670,0,"module") -#20672=@"loc,{#10000},22,22,22,27" -locations_default(#20672,#10000,22,22,22,27) +properties(#20671,#20669,0,0,"type: ""json""") +#20672=@"loc,{#10000},17,38,17,49" +locations_default(#20672,#10000,17,38,17,49) hasLocation(#20671,#20672) -regexp_const_value(#20671,"module") #20673=* -exprs(#20673,8,#20669,-10,"{ type: ""json"" }") -#20674=@"loc,{#10000},22,37,22,52" -locations_default(#20674,#10000,22,37,22,52) -hasLocation(#20673,#20674) -enclosing_stmt(#20673,#20669) +exprs(#20673,0,#20671,0,"type") +hasLocation(#20673,#20341) +enclosing_stmt(#20673,#20665) expr_containers(#20673,#20001) +literals("type","type",#20673) +#20674=* +exprs(#20674,4,#20671,1,"""json""") +hasLocation(#20674,#20345) +enclosing_stmt(#20674,#20665) +expr_containers(#20674,#20001) +literals("json","""json""",#20674) #20675=* -properties(#20675,#20673,0,0,"type: ""json""") -#20676=@"loc,{#10000},22,39,22,50" -locations_default(#20676,#10000,22,39,22,50) +regexpterm(#20675,14,#20674,0,"json") +#20676=@"loc,{#10000},17,45,17,48" +locations_default(#20676,#10000,17,45,17,48) hasLocation(#20675,#20676) +regexp_const_value(#20675,"json") #20677=* -exprs(#20677,96,#20669,0,"* as v5") -#20678=@"loc,{#10000},22,8,22,14" -locations_default(#20678,#10000,22,8,22,14) -hasLocation(#20677,#20678) -enclosing_stmt(#20677,#20669) +exprs(#20677,83,#20665,0,"v2") +hasLocation(#20677,#20329) +enclosing_stmt(#20677,#20665) expr_containers(#20677,#20001) +#20678=* +exprs(#20678,0,#20677,0,"v2") +hasLocation(#20678,#20329) +enclosing_stmt(#20678,#20665) +expr_containers(#20678,#20001) +literals("v2","v2",#20678) #20679=* -exprs(#20679,0,#20677,1,"v5") -hasLocation(#20679,#20427) -enclosing_stmt(#20679,#20669) +exprs(#20679,78,#20677,1,"v2") +hasLocation(#20679,#20329) +enclosing_stmt(#20679,#20665) expr_containers(#20679,#20001) -literals("v5","v5",#20679) +literals("v2","v2",#20679) +decl(#20679,#20487) +typedecl(#20679,#20490) +namespacedecl(#20679,#20493) #20680=* -stmts(#20680,22,#20001,17,"const v ... "" } });") -hasLocation(#20680,#20053) +stmts(#20680,27,#20001,13,"import ... son"" };") +hasLocation(#20680,#20041) stmt_containers(#20680,#20001) #20681=* -exprs(#20681,64,#20680,0,"v6 = im ... n"" } })") -#20682=@"loc,{#10000},24,7,24,57" -locations_default(#20682,#10000,24,7,24,57) -hasLocation(#20681,#20682) +exprs(#20681,4,#20680,-1,"""module""") +hasLocation(#20681,#20357) enclosing_stmt(#20681,#20680) expr_containers(#20681,#20001) -#20683=* -exprs(#20683,78,#20681,0,"v6") -hasLocation(#20683,#20449) -enclosing_stmt(#20683,#20680) -expr_containers(#20683,#20001) -literals("v6","v6",#20683) -decl(#20683,#20495) +literals("module","""module""",#20681) +#20682=* +regexpterm(#20682,14,#20681,0,"module") +#20683=@"loc,{#10000},18,17,18,22" +locations_default(#20683,#10000,18,17,18,22) +hasLocation(#20682,#20683) +regexp_const_value(#20682,"module") #20684=* -exprs(#20684,99,#20681,1,"import( ... n"" } })") -#20685=@"loc,{#10000},24,12,24,57" -locations_default(#20685,#10000,24,12,24,57) +exprs(#20684,8,#20680,-10,"{ type: ""json"" }") +#20685=@"loc,{#10000},18,32,18,47" +locations_default(#20685,#10000,18,32,18,47) hasLocation(#20684,#20685) enclosing_stmt(#20684,#20680) expr_containers(#20684,#20001) #20686=* -exprs(#20686,4,#20684,0,"""module""") -hasLocation(#20686,#20457) -enclosing_stmt(#20686,#20680) -expr_containers(#20686,#20001) -literals("module","""module""",#20686) -#20687=* -regexpterm(#20687,14,#20686,0,"module") -#20688=@"loc,{#10000},24,20,24,25" -locations_default(#20688,#10000,24,20,24,25) -hasLocation(#20687,#20688) -regexp_const_value(#20687,"module") +properties(#20686,#20684,0,0,"type: ""json""") +#20687=@"loc,{#10000},18,34,18,45" +locations_default(#20687,#10000,18,34,18,45) +hasLocation(#20686,#20687) +#20688=* +exprs(#20688,0,#20686,0,"type") +hasLocation(#20688,#20363) +enclosing_stmt(#20688,#20680) +expr_containers(#20688,#20001) +literals("type","type",#20688) #20689=* -exprs(#20689,8,#20684,1,"{ asser ... on"" } }") -#20690=@"loc,{#10000},24,29,24,56" -locations_default(#20690,#10000,24,29,24,56) -hasLocation(#20689,#20690) +exprs(#20689,4,#20686,1,"""json""") +hasLocation(#20689,#20367) enclosing_stmt(#20689,#20680) expr_containers(#20689,#20001) -#20691=* -properties(#20691,#20689,0,0,"assert: ... json"" }") -#20692=@"loc,{#10000},24,31,24,54" -locations_default(#20692,#10000,24,31,24,54) -hasLocation(#20691,#20692) +literals("json","""json""",#20689) +#20690=* +regexpterm(#20690,14,#20689,0,"json") +#20691=@"loc,{#10000},18,41,18,44" +locations_default(#20691,#10000,18,41,18,44) +hasLocation(#20690,#20691) +regexp_const_value(#20690,"json") +#20692=* +exprs(#20692,84,#20680,0,"v3") +hasLocation(#20692,#20353) +enclosing_stmt(#20692,#20680) +expr_containers(#20692,#20001) #20693=* -exprs(#20693,0,#20691,0,"assert") -hasLocation(#20693,#20463) +exprs(#20693,78,#20692,1,"v3") +hasLocation(#20693,#20353) enclosing_stmt(#20693,#20680) expr_containers(#20693,#20001) -literals("assert","assert",#20693) +literals("v3","v3",#20693) +decl(#20693,#20488) +typedecl(#20693,#20491) +namespacedecl(#20693,#20494) #20694=* -exprs(#20694,8,#20691,1,"{ type: ""json"" }") -#20695=@"loc,{#10000},24,39,24,54" -locations_default(#20695,#10000,24,39,24,54) -hasLocation(#20694,#20695) -enclosing_stmt(#20694,#20680) -expr_containers(#20694,#20001) +stmts(#20694,30,#20001,14,"export ... son"" };") +hasLocation(#20694,#20045) +stmt_containers(#20694,#20001) +#20695=* +exprs(#20695,4,#20694,-2,"""module""") +hasLocation(#20695,#20383) +enclosing_stmt(#20695,#20694) +expr_containers(#20695,#20001) +literals("module","""module""",#20695) #20696=* -properties(#20696,#20694,0,0,"type: ""json""") -#20697=@"loc,{#10000},24,41,24,52" -locations_default(#20697,#10000,24,41,24,52) +regexpterm(#20696,14,#20695,0,"module") +#20697=@"loc,{#10000},20,21,20,26" +locations_default(#20697,#10000,20,21,20,26) hasLocation(#20696,#20697) +regexp_const_value(#20696,"module") #20698=* -exprs(#20698,0,#20696,0,"type") -hasLocation(#20698,#20469) -enclosing_stmt(#20698,#20680) +exprs(#20698,8,#20694,-10,"{ type: ""json"" }") +#20699=@"loc,{#10000},20,36,20,51" +locations_default(#20699,#10000,20,36,20,51) +hasLocation(#20698,#20699) +enclosing_stmt(#20698,#20694) expr_containers(#20698,#20001) -literals("type","type",#20698) -#20699=* -exprs(#20699,4,#20696,1,"""json""") -hasLocation(#20699,#20473) -enclosing_stmt(#20699,#20680) -expr_containers(#20699,#20001) -literals("json","""json""",#20699) #20700=* -regexpterm(#20700,14,#20699,0,"json") -#20701=@"loc,{#10000},24,48,24,51" -locations_default(#20701,#10000,24,48,24,51) +properties(#20700,#20698,0,0,"type: ""json""") +#20701=@"loc,{#10000},20,38,20,49" +locations_default(#20701,#10000,20,38,20,49) hasLocation(#20700,#20701) -regexp_const_value(#20700,"json") #20702=* -entry_cfg_node(#20702,#20001) -#20703=@"loc,{#10000},1,1,1,0" -locations_default(#20703,#10000,1,1,1,0) -hasLocation(#20702,#20703) +exprs(#20702,0,#20700,0,"type") +hasLocation(#20702,#20389) +enclosing_stmt(#20702,#20694) +expr_containers(#20702,#20001) +literals("type","type",#20702) +#20703=* +exprs(#20703,4,#20700,1,"""json""") +hasLocation(#20703,#20393) +enclosing_stmt(#20703,#20694) +expr_containers(#20703,#20001) +literals("json","""json""",#20703) #20704=* -exit_cfg_node(#20704,#20001) -hasLocation(#20704,#20483) -successor(#20680,#20683) -successor(#20686,#20684) -successor(#20684,#20681) -successor(#20683,#20686) -successor(#20681,#20704) -successor(#20669,#20670) -successor(#20677,#20679) -successor(#20679,#20680) -successor(#20670,#20677) -successor(#20661,#20662) -successor(#20662,#20669) -successor(#20650,#20651) -successor(#20658,#20659) -successor(#20660,#20661) -successor(#20659,#20660) -successor(#20651,#20658) -successor(#20640,#20650) -successor(#20629,#20640) -successor(#20618,#20629) -successor(#20610,#20618) -successor(#20596,#20600) -successor(#20602,#20606) -successor(#20607,#20604) -successor(#20606,#20607) -successor(#20604,#20598) -successor(#20600,#20602) -successor(#20598,#20610) -successor(#20591,#20596) -successor(#20567,#20570) -successor(#20573,#20571) -successor(#20571,#20568) -successor(#20570,#20573) -successor(#20568,#20591) -successor(#20556,#20557) -successor(#20564,#20566) +regexpterm(#20704,14,#20703,0,"json") +#20705=@"loc,{#10000},20,45,20,48" +locations_default(#20705,#10000,20,45,20,48) +hasLocation(#20704,#20705) +regexp_const_value(#20704,"json") +#20706=* +exprs(#20706,86,#20694,0,"v4") +hasLocation(#20706,#20377) +enclosing_stmt(#20706,#20694) +expr_containers(#20706,#20001) +#20707=* +exprs(#20707,0,#20706,0,"v4") +hasLocation(#20707,#20377) +enclosing_stmt(#20707,#20694) +expr_containers(#20707,#20001) +literals("v4","v4",#20707) +#20708=* +exprs(#20708,0,#20706,1,"v4") +hasLocation(#20708,#20377) +enclosing_stmt(#20708,#20694) +expr_containers(#20708,#20001) +literals("v4","v4",#20708) +#20709=* +stmts(#20709,28,#20001,15,"export ... son"" };") +hasLocation(#20709,#20047) +stmt_containers(#20709,#20001) +#20710=* +exprs(#20710,4,#20709,0,"""module""") +hasLocation(#20710,#20405) +enclosing_stmt(#20710,#20709) +expr_containers(#20710,#20001) +literals("module","""module""",#20710) +#20711=* +regexpterm(#20711,14,#20710,0,"module") +#20712=@"loc,{#10000},21,16,21,21" +locations_default(#20712,#10000,21,16,21,21) +hasLocation(#20711,#20712) +regexp_const_value(#20711,"module") +#20713=* +exprs(#20713,8,#20709,-10,"{ type: ""json"" }") +#20714=@"loc,{#10000},21,31,21,46" +locations_default(#20714,#10000,21,31,21,46) +hasLocation(#20713,#20714) +enclosing_stmt(#20713,#20709) +expr_containers(#20713,#20001) +#20715=* +properties(#20715,#20713,0,0,"type: ""json""") +#20716=@"loc,{#10000},21,33,21,44" +locations_default(#20716,#10000,21,33,21,44) +hasLocation(#20715,#20716) +#20717=* +exprs(#20717,0,#20715,0,"type") +hasLocation(#20717,#20411) +enclosing_stmt(#20717,#20709) +expr_containers(#20717,#20001) +literals("type","type",#20717) +#20718=* +exprs(#20718,4,#20715,1,"""json""") +hasLocation(#20718,#20415) +enclosing_stmt(#20718,#20709) +expr_containers(#20718,#20001) +literals("json","""json""",#20718) +#20719=* +regexpterm(#20719,14,#20718,0,"json") +#20720=@"loc,{#10000},21,40,21,43" +locations_default(#20720,#10000,21,40,21,43) +hasLocation(#20719,#20720) +regexp_const_value(#20719,"json") +#20721=* +stmts(#20721,30,#20001,16,"export ... son"" };") +hasLocation(#20721,#20049) +stmt_containers(#20721,#20001) +#20722=* +exprs(#20722,4,#20721,-2,"""module""") +hasLocation(#20722,#20431) +enclosing_stmt(#20722,#20721) +expr_containers(#20722,#20001) +literals("module","""module""",#20722) +#20723=* +regexpterm(#20723,14,#20722,0,"module") +#20724=@"loc,{#10000},22,22,22,27" +locations_default(#20724,#10000,22,22,22,27) +hasLocation(#20723,#20724) +regexp_const_value(#20723,"module") +#20725=* +exprs(#20725,8,#20721,-10,"{ type: ""json"" }") +#20726=@"loc,{#10000},22,37,22,52" +locations_default(#20726,#10000,22,37,22,52) +hasLocation(#20725,#20726) +enclosing_stmt(#20725,#20721) +expr_containers(#20725,#20001) +#20727=* +properties(#20727,#20725,0,0,"type: ""json""") +#20728=@"loc,{#10000},22,39,22,50" +locations_default(#20728,#10000,22,39,22,50) +hasLocation(#20727,#20728) +#20729=* +exprs(#20729,0,#20727,0,"type") +hasLocation(#20729,#20437) +enclosing_stmt(#20729,#20721) +expr_containers(#20729,#20001) +literals("type","type",#20729) +#20730=* +exprs(#20730,4,#20727,1,"""json""") +hasLocation(#20730,#20441) +enclosing_stmt(#20730,#20721) +expr_containers(#20730,#20001) +literals("json","""json""",#20730) +#20731=* +regexpterm(#20731,14,#20730,0,"json") +#20732=@"loc,{#10000},22,46,22,49" +locations_default(#20732,#10000,22,46,22,49) +hasLocation(#20731,#20732) +regexp_const_value(#20731,"json") +#20733=* +exprs(#20733,96,#20721,0,"* as v5") +#20734=@"loc,{#10000},22,8,22,14" +locations_default(#20734,#10000,22,8,22,14) +hasLocation(#20733,#20734) +enclosing_stmt(#20733,#20721) +expr_containers(#20733,#20001) +#20735=* +exprs(#20735,0,#20733,1,"v5") +hasLocation(#20735,#20427) +enclosing_stmt(#20735,#20721) +expr_containers(#20735,#20001) +literals("v5","v5",#20735) +#20736=* +stmts(#20736,22,#20001,17,"const v ... "" } });") +hasLocation(#20736,#20053) +stmt_containers(#20736,#20001) +#20737=* +exprs(#20737,64,#20736,0,"v6 = im ... n"" } })") +#20738=@"loc,{#10000},24,7,24,57" +locations_default(#20738,#10000,24,7,24,57) +hasLocation(#20737,#20738) +enclosing_stmt(#20737,#20736) +expr_containers(#20737,#20001) +#20739=* +exprs(#20739,78,#20737,0,"v6") +hasLocation(#20739,#20449) +enclosing_stmt(#20739,#20736) +expr_containers(#20739,#20001) +literals("v6","v6",#20739) +decl(#20739,#20495) +#20740=* +exprs(#20740,99,#20737,1,"import( ... n"" } })") +#20741=@"loc,{#10000},24,12,24,57" +locations_default(#20741,#10000,24,12,24,57) +hasLocation(#20740,#20741) +enclosing_stmt(#20740,#20736) +expr_containers(#20740,#20001) +#20742=* +exprs(#20742,4,#20740,0,"""module""") +hasLocation(#20742,#20457) +enclosing_stmt(#20742,#20736) +expr_containers(#20742,#20001) +literals("module","""module""",#20742) +#20743=* +regexpterm(#20743,14,#20742,0,"module") +#20744=@"loc,{#10000},24,20,24,25" +locations_default(#20744,#10000,24,20,24,25) +hasLocation(#20743,#20744) +regexp_const_value(#20743,"module") +#20745=* +exprs(#20745,8,#20740,1,"{ asser ... on"" } }") +#20746=@"loc,{#10000},24,29,24,56" +locations_default(#20746,#10000,24,29,24,56) +hasLocation(#20745,#20746) +enclosing_stmt(#20745,#20736) +expr_containers(#20745,#20001) +#20747=* +properties(#20747,#20745,0,0,"assert: ... json"" }") +#20748=@"loc,{#10000},24,31,24,54" +locations_default(#20748,#10000,24,31,24,54) +hasLocation(#20747,#20748) +#20749=* +exprs(#20749,0,#20747,0,"assert") +hasLocation(#20749,#20463) +enclosing_stmt(#20749,#20736) +expr_containers(#20749,#20001) +literals("assert","assert",#20749) +#20750=* +exprs(#20750,8,#20747,1,"{ type: ""json"" }") +#20751=@"loc,{#10000},24,39,24,54" +locations_default(#20751,#10000,24,39,24,54) +hasLocation(#20750,#20751) +enclosing_stmt(#20750,#20736) +expr_containers(#20750,#20001) +#20752=* +properties(#20752,#20750,0,0,"type: ""json""") +#20753=@"loc,{#10000},24,41,24,52" +locations_default(#20753,#10000,24,41,24,52) +hasLocation(#20752,#20753) +#20754=* +exprs(#20754,0,#20752,0,"type") +hasLocation(#20754,#20469) +enclosing_stmt(#20754,#20736) +expr_containers(#20754,#20001) +literals("type","type",#20754) +#20755=* +exprs(#20755,4,#20752,1,"""json""") +hasLocation(#20755,#20473) +enclosing_stmt(#20755,#20736) +expr_containers(#20755,#20001) +literals("json","""json""",#20755) +#20756=* +regexpterm(#20756,14,#20755,0,"json") +#20757=@"loc,{#10000},24,48,24,51" +locations_default(#20757,#10000,24,48,24,51) +hasLocation(#20756,#20757) +regexp_const_value(#20756,"json") +#20758=* +entry_cfg_node(#20758,#20001) +#20759=@"loc,{#10000},1,1,1,0" +locations_default(#20759,#10000,1,1,1,0) +hasLocation(#20758,#20759) +#20760=* +exit_cfg_node(#20760,#20001) +hasLocation(#20760,#20483) +successor(#20736,#20739) +successor(#20742,#20740) +successor(#20740,#20737) +successor(#20739,#20742) +successor(#20737,#20760) +successor(#20721,#20722) +successor(#20733,#20735) +successor(#20735,#20736) +successor(#20722,#20733) +successor(#20709,#20710) +successor(#20710,#20721) +successor(#20694,#20695) +successor(#20706,#20707) +successor(#20708,#20709) +successor(#20707,#20708) +successor(#20695,#20706) +successor(#20680,#20694) +successor(#20665,#20680) +successor(#20650,#20665) +successor(#20638,#20650) +successor(#20624,#20628) +successor(#20630,#20634) +successor(#20635,#20632) +successor(#20634,#20635) +successor(#20632,#20626) +successor(#20628,#20630) +successor(#20626,#20638) +successor(#20619,#20624) +successor(#20595,#20598) +successor(#20601,#20599) +successor(#20599,#20596) +successor(#20598,#20601) +successor(#20596,#20619) +successor(#20580,#20581) +successor(#20592,#20594) +successor(#20594,#20595) +successor(#20581,#20592) +successor(#20568,#20569) +successor(#20569,#20580) +successor(#20553,#20554) +successor(#20565,#20566) +successor(#20567,#20568) successor(#20566,#20567) -successor(#20557,#20564) -successor(#20548,#20549) -successor(#20549,#20556) -successor(#20537,#20538) -successor(#20545,#20546) -successor(#20547,#20548) -successor(#20546,#20547) -successor(#20538,#20545) -successor(#20527,#20537) -successor(#20516,#20527) -successor(#20505,#20516) -successor(#20497,#20505) -successor(#20648,#20497) -successor(#20637,#20648) -successor(#20626,#20637) -successor(#20535,#20626) -successor(#20524,#20535) -successor(#20513,#20524) -successor(#20702,#20513) +successor(#20554,#20565) +successor(#20539,#20553) +successor(#20524,#20539) +successor(#20509,#20524) +successor(#20497,#20509) +successor(#20692,#20497) +successor(#20677,#20692) +successor(#20662,#20677) +successor(#20551,#20662) +successor(#20536,#20551) +successor(#20521,#20536) +successor(#20758,#20521) numlines(#10000,24,18,2) filetype(#10000,"typescript") diff --git a/javascript/ql/integration-tests/diagnostics/internal-error/fail-native.cjs b/javascript/ql/integration-tests/diagnostics/internal-error/fail-native.cjs new file mode 100644 index 000000000000..6dd5449b1acb --- /dev/null +++ b/javascript/ql/integration-tests/diagnostics/internal-error/fail-native.cjs @@ -0,0 +1,7 @@ +const childProcess = require("node:child_process"); +const spawn = childProcess.spawn; +childProcess.spawn = function (...args) { + const child = spawn.apply(this, args); + child.kill(); + return child; +}; diff --git a/javascript/ql/integration-tests/diagnostics/internal-error/test.py b/javascript/ql/integration-tests/diagnostics/internal-error/test.py index 25a95e763ee8..c9e36473f627 100644 --- a/javascript/ql/integration-tests/diagnostics/internal-error/test.py +++ b/javascript/ql/integration-tests/diagnostics/internal-error/test.py @@ -1,2 +1,6 @@ -def test(codeql, javascript): - codeql.database.create(_assert_failure=True) +import os + + +def test(codeql, javascript, cwd): + os.environ["NODE_OPTIONS"] = f'--require "{cwd / "fail-native.cjs"}"' + codeql.database.create(source_root="src", _assert_failure=True) diff --git a/javascript/ql/test/library-tests/TypeScript/SourceImports/test.expected b/javascript/ql/test/library-tests/TypeScript/SourceImports/test.expected new file mode 100644 index 000000000000..769e9dfee998 --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/SourceImports/test.expected @@ -0,0 +1,5 @@ +imports +| tst.ts:1:1:1:40 | import ... .wasm"; | +references +| tst.ts:1:15:1:18 | wasm | +| tst.ts:3:1:3:4 | wasm | diff --git a/javascript/ql/test/library-tests/TypeScript/SourceImports/test.ql b/javascript/ql/test/library-tests/TypeScript/SourceImports/test.ql new file mode 100644 index 000000000000..0997989d37ec --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/SourceImports/test.ql @@ -0,0 +1,5 @@ +import javascript + +query predicate imports(ImportDeclaration decl) { any() } + +query predicate references(VarRef ref) { any() } diff --git a/javascript/ql/test/library-tests/TypeScript/SourceImports/tst.ts b/javascript/ql/test/library-tests/TypeScript/SourceImports/tst.ts new file mode 100644 index 000000000000..d1c1fa88139a --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/SourceImports/tst.ts @@ -0,0 +1,3 @@ +import source wasm from "./module.wasm"; + +wasm; diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index 684bcbb8c3ba..1410e12330bb 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -84,7 +84,7 @@ CodeQLPackInfo = provider( "pack_prefix": "A prefix to add to all paths, IF the user requests so. We omit it for local installation targets of single packs (but not pack groups)", "files": "PackageFilegroupInfo provider with list of all files in this pack (CODEQL_PLATFORM in paths unresolved)", "zips": "A _ZipInfo provider to include in the pack, (CODEQL_PLATFORM unresolved).", - "arch_overrides": "A list of files that should be included in the arch-specific bit, even though the path doesn't contain CODEQL_PLATFORM.", + "arch_overrides": "Files or imported zip prefixes to include in the arch-specific bit, even without CODEQL_PLATFORM in their paths.", }, ) @@ -220,7 +220,7 @@ _codeql_pack_info = rule( allow_files = [".zip"], ), "prefix": attr.string(doc = "Prefix to add to all files."), - "arch_overrides": attr.string_list(doc = "A list of files that should be included in the arch package regardless of the path, specify the path _without_ `prefix`."), + "arch_overrides": attr.string_list(doc = "Files or imported zip prefixes to include in the arch package regardless of the path, specified _without_ `prefix`."), }, provides = [CodeQLPackInfo], ) @@ -293,6 +293,8 @@ def _codeql_pack_group_extract_zips_impl(ctx): # for each zip file, resolve whether it's filtered out or not by the current kind, and add the pack prefix for zip, zip_prefix in src.zips.zips_to_prefixes.items(): zip_kind, expanded_prefix = _expand_path(paths.join(prefix, zip_prefix), platform) + if zip_kind == "common" and zip_prefix in src.arch_overrides: + zip_kind = "arch" if include_all_files or zip_kind == ctx.attr.kind: zips_to_prefixes[zip] = expanded_prefix @@ -465,13 +467,13 @@ def codeql_pack( The distinction between arch-specific and common contents is made based on whether the paths (including possible prefixes added by rules) contain the special `{CODEQL_PLATFORM}` placeholder, which in case it is present will also be replaced by the appropriate platform (`linux64`, `linux-arm64`, `win64` or `osx64`). - Specific file paths can be placed in the arch-specific package by adding them to `arch_overrides`, even if their - path doesn't contain the `CODEQL_PLATFORM` placeholder. + Specific file paths or imported zip prefixes can be placed in the arch-specific package by adding them to + `arch_overrides`, even if their path doesn't contain the `CODEQL_PLATFORM` placeholder. The codeql pack rules will expand the `{CODEQL_PLATFORM}` marker in paths, and use that to split the files into a common and an arch-specific part. This placeholder will be replaced by the appropriate platform (`linux64`, `linux-arm64`, `win64` or `osx64`). - `arch_overrides` is a list of files that should be included in the arch-specific bits of the pack, even if their path doesn't - contain the `{CODEQL_PLATFORM}` marker. + `arch_overrides` is a list of files or imported zip prefixes that should be included in the arch-specific bits of the + pack, even if their path doesn't contain the `{CODEQL_PLATFORM}` marker. All files in the pack will be prefixed with `name`, unless `pack_prefix` is set, then is used instead. This rule also provides a convenient installer target named `-installer`, with a path governed by `install_dest`,