Migrating Babel from CommonJS to ESM

@NicoloRibaudo
Migrating Babel from
CommonJS to ESM
NICOLÒ RIBAUDO
@nicolo-ribaudo
@NicoloRibaudo hello@nicr.dev

@NicoloRibaudo
Once upon a time …
3

@NicoloRibaudo
… there was a little JavaScript library, written using the shiny new
import / export syntax and published to npm compiled to CommonJS.
Once upon a time …
4
import { parse } from "babylon";
export function transform(code) {
let ast = parse(code);
return es6to5(ast);
}
"use strict";
exports.__esModule = true;
exports.transform = transform;
var _babylon = require("babylon");
function transform(code) {
let ast = _babylon.parse(code);
return es6to5(ast);
}

@NicoloRibaudo
Until one day, suddenly …
5

@NicoloRibaudo
6

@NicoloRibaudo
7

@NicoloRibaudo
Our adventure begins here.
8

@NicoloRibaudo
Chapter 1:
Introduction
9

@NicoloRibaudo
What is Babel?
10

@NicoloRibaudo
What is Babel?
11
Babel is a build-time
compiler
JavaScript

@NicoloRibaudo
What is Babel?
12
compiler
JavaScript

@NicoloRibaudo
What is Babel?
13
devtool

@NicoloRibaudo
What is Babel?
14
devtool
conﬁgurable

@NicoloRibaudo 15
// babel.config.js
module.exports = function () {
return {
targets: [
"last 3 versions",
"not ie"
],
ignore: ["**/*.test.js"]
};
};
$ npx babel src --out-dir lib
"Generate code that is compatible
with these browsers"
"Don't compile test ﬁles"

@NicoloRibaudo
What is Babel?
16
devtool
pluggable

@NicoloRibaudo 17
// babel-plugin-hello.js
const { types: t } = require("@babel/core");
module.exports = () => ({
visitor: {
StringLiteral(path) {
path.replaceWith(
t.stringLiteral("Hello World!")
);
},
},
});
$ npx babel src --out-dir lib
Apply some transformation
to the parsed code,
potentially relying on
Babel-provided AST utilities

@NicoloRibaudo
What is Babel?
18
devtool
n embeddable

@NicoloRibaudo 19
// webpack.config.js
module.exports = {
entry: "./src/main.js",
output: "./out/bundle.js",
module: {
rules: [
{
test: /.js|.ts|.jsx/,
use: "babel-loader",
},
],
},
};
// rollup.config.js
import {
babel
} from "@rollup/plugin-babel";
export default {
input: "src/index.js",
output: {
dir: "output",
format: "es",
},
plugins: [ babel() ],
};
● Webpack
● Rollup
● Parcel
● Vite
● ESLint
● Prettier
● …

@NicoloRibaudo
What is Babel?
20
library
JavaScript

@NicoloRibaudo 21
const babel = require("@babel/core");
const fs = require("fs/promises");
babel
.transformFileAsync("./main.js")
.then(({ code }) =>
fs.writeFile("./main.out.js", code)
).catch(err =>
console.error("Cannot compile!", err)
);
const parser = require("@babl/parser");
const generator = require("@babl/generator");
const ast = parser.parse("const a = 1;");
ast.program.body[0].declarations[0]
.id.name = "b";
const code = generator.default(ast);
console.log(code); // "const b = 1;"

@NicoloRibaudo
ESM challenges
22

@NicoloRibaudo
ESM challenges
23
#1 — It cannot be synchronously imported from CommonJS in Node.js
// SyntaxError in CommonJS
import "./module.mjs";
// Error [ERR_REQUIRE_ESM]: require() of ES Module not supported
require("./module.mjs");
// This works, but it's asynchronous
import("./module.mjs"); // Promise

@NicoloRibaudo
ESM challenges
24
#2 — It cannot be synchronously imported dynamically or lazily
// CommonJS
function loadIt() {
require("./module.cjs");
}
// ES Modules - SyntaxError // ES Modules - async
function loadIt() { async function loadIt() {
import "./module.cjs"; await import("./module.cjs");
} }

@NicoloRibaudo
ESM challenges
25
#3 — ESM-compiled-to-CJS has a diﬀerent interface from native ESM
import obj from "./esm-compiled-to-cjs.js";
console.log(obj);
// { __esModule: true, default: [Function: A], namedExport: "foo" }

@NicoloRibaudo
ESM challenges
26
#4 — It doesn't integrate with tools that virtualize require and CJS loading
● mocking
● on-the-ﬂy transpilation
● other bad things
🧙

@NicoloRibaudo
Internal vs External
code
27

@NicoloRibaudo
28
Babel's source
● Written by Babel contributors
● Used by all Babel users
Babel's tests
● Written by Babel contributors
● Used by Babel contributors
Conﬁguration ﬁles
● Written by (almost) all Babel users
Plugins
● Written by a limited number of
developers

@NicoloRibaudo
29

@NicoloRibaudo
30

@NicoloRibaudo
Chapter 2:
Making Babel
asynchronous
31

@NicoloRibaudo
The async/await virus
32
function transform(code, opts) {
const config = loadFullConfig(opts);
// ...
function loadFullConfig(inputOpts) {
const result = loadPrivatePartialConfig(inputOpts);
// ...
function loadPrivatePartialConfig(inputOpts) {
// ...
const configChain = buildRootChain(args, context);
// ...
function buildRootChain(opts, context) {
// ...
configFile = loadOneConfig(ROOT_FILENAMES, dirname);
// ...
function loadOneConfig(filenames, dirname) {
// ...
const config = readConfigCode(filepath);
// ...
function readConfigCode(filepath) {
// ...
options = loadCodeDefault(filepath);
// ...
function loadCodeDefault(filepath) {
const module = require(filepath);
// ...

@NicoloRibaudo
33
// ...
// ...
// ...
// ...
// ...
// ...
// ...
// ...
// ...
// ...
const module = await import(filepath);
// ...

@NicoloRibaudo
34
// ...
// ...
// ...
// ...
// ...
// ...
// ...
// ...
// ...
// ...
async function loadCodeDefault(filepath) {
// ...

@NicoloRibaudo
35
// ...
// ...
// ...
// ...
// ...
// ...
// ...
// ...
// ...
options = await loadCodeDefault(filepath);
// ...
// ...

@NicoloRibaudo
36
async function transform(code, opts) {
const config = await loadFullConfig(opts);
// ...
async function loadFullConfig(inputOpts) {
const result = await loadPrivatePartialConfig(inputOpts);
// ...
async function loadPrivatePartialConfig(inputOpts) {
// ...
const configChain = await buildRootChain(args, context);
// ...
async function buildRootChain(opts, context) {
// ...
configFile = await loadOneConfig(ROOT_FILENAMES, dirname);
// ...
async function loadOneConfig(filenames, dirname) {
// ...
const config = await readConfigCode(filepath);
// ...
async function readConfigCode(filepath) {
// ...
options = await loadCodeDefault(filepath);
// ...
// ...

@NicoloRibaudo
Preserving Babel's sync API
37
● Preserves backward compatibility
● The asynchronous API is only necessary when loading ESM ﬁles
function transform(code, opts) async function transformAsync(code, opts)

@NicoloRibaudo
38
function transform(code, opts)
if (isCommonJS(filepath)) {
module = require(filepath);
} else {
throw new Error("Unsupported ESM config!");
}
async function transformAsync(code, opts)
async function loadFullConfig(inputOpts) {
async function loadPrivatePartialConfig(inputOpts)
async function buildRootChain(opts, context) {
async function loadOneConfig(filenames, dirname) {
async function readConfigCode(filepath) {
} else {
module = await import(filepath);
}
{

@NicoloRibaudo
39
Can we have a single implementation, capable of running both synchronously
and asynchronously?

@NicoloRibaudo
40
Can we have a single implementation, capable of running both synchronously
and asynchronously?
Callbacks?
function loadCodeDefault(filepath, callback) {
try { callback(null, require(filepath)); }
catch (err) { callback(err); }
} else {
import(filepath).then(
module => callback(null, module),
err => callback(err)
);
}
}

@NicoloRibaudo
gensync: abstracting the
41
gensync: abstracting the ???
Me trying to
prepare these
slides
Me spending too
much time looking
for a word

@NicoloRibaudo
42
https://github.com/loganfsmyth/gensync

@NicoloRibaudo
43
const readJSON = async function (filepath) {
const contents = await readFile(filepath, "utf8");
return JSON.parse(contents);
};

@NicoloRibaudo
44
};
const readJSON = gensync(function* (filepath) {
const contents = yield* readFile(filepath, "utf8");
});

@NicoloRibaudo
45
};
});
const json = readJSON.sync("package.json");
// or
const json = await readJSON.async("package.json");

@NicoloRibaudo
46
};
});
// or
const readFile = gensync({
sync: fs.readFileSync,
errback: fs.readFile,
});

@NicoloRibaudo
47
};
});
// or
const readFile = gensync({
sync: fs.readFileSync,
errback: fs.readFile,
});
You can deﬁne any utility that
branches on the execution model:
const isAsync = gensync({
sync: () => false,
async: async () => true,
});

@NicoloRibaudo 48
const transform = gensync(function* (code, opts) {
const config = yield* loadFullConfig(opts);
// ...
function* loadFullConfig(inputOpts) {
const result = yield* loadPrivatePartialConfig(inputOpts);
// ...
function* loadPrivatePartialConfig(opts) {
// ...
const configChain = yield* buildRootChain(args, context);
// ...
function* buildRootChain(opts, context) {
// ...
configFile = yield* loadOneConfig(ROOT_FILENAMES, dirname);
// ...
function* loadOneConfig(filenames, dirname) {
// ...
const config = yield* readConfigCode(filepath);
// ...
}
function* readConfigCode(filepath) {
// ...
options = yield* loadCodeDefault(filepath);
// ...
}

@NicoloRibaudo 49
const transform = gensync(function* (code, opts) {
const config = yield* loadFullConfig(opts);
// ...
export const transform = transform.errback;
export const transformSync = transform.sync;
export const transformAsync = transform.async;

@NicoloRibaudo 50
} else {
throw new Error("Unsupported ESM!");
}
} else {
}

@NicoloRibaudo 51
} else {
}
} else {
}
function* loadCodeDefault(filepath) {
} else if (yield* isAsync()) {
module = yield* wait(import(filepath));
} else {
}
// ...
}
const wait = gensync({
sync: x => x,
async: x => x,
});

@NicoloRibaudo 52

@NicoloRibaudo
.mjs conﬁg ﬁles and plugins :)
53

@NicoloRibaudo
Chapter 3:
Making Babel
synchronous again
54

@NicoloRibaudo
Making Babel synchronous again
55
Sometimes you can force😇 asynchronous APIs on your
users — sometimes you can't.

@NicoloRibaudo
Making Babel synchronous again
56
Sometimes you can force😇 asynchronous APIs on your
users — sometimes you can't.
@babel/eslint-parser is an ESLint parser to support experimental syntax;
@babel/register hooks into Node.js' require() to compile ﬁles on-the-ﬂy.

@NicoloRibaudo
Worker and Atomics to the rescue
57

@NicoloRibaudo
Main thread
function doSomethingSync() {
const signal = new Int32Array(new SharedArrayBuffer(4));
const { port1, port2 } = new MessageChannel();
// sleep
Atomics.wait(signal, 0, 0);
const { result } = receiveMessageOnPort(port2);
return result;
}
58
Worker thread
addListener("message", () => {
let result =
await doSomethingAsync();
port1.postMessage({ result });
Atomics.store(signal, 0, 1);
Atomics.notify(signal, 0);
});
signal, port1
{ payload: /* ... */ }
{ result: /* ... */ }

@NicoloRibaudo
59
More details:
https://giuseppegurgone.com/synchronizing-async-functions

@NicoloRibaudo
Main thread
function doSomethingSync() {
● Wait (sleep) until SAB's
contents change
● Read the received result
return result;
}
60
Worker thread
addListener("message", () => {
let result =
await doSomethingAsync();
● Change SAB's contents
● Wake up the main thread
});
SharedArrayBuffer[ 0x00 0x00 0x00 0x00 ]
{ payload: /* ... */ }
{ result: /* ... */ }

@NicoloRibaudo
Main thread
1. Create the worker
const { Worker, SHARE_ENV } = require("worker_threads");
const worker = new Worker("./path/to/worker.js", {
env: SHARE_ENV,
});
61

@NicoloRibaudo
Main thread, doSomethingSync
2. Delegate the task to the worker
const { MessageChannel } = require("worker_threads");
const signal = new Int32Array(new SharedArrayBuffer(4));
const { port1, port2 } = new MessageChannel();
worker.postMessage({ signal, port: port1, payload }, [port1]);
Atomics.wait(signal, 0, 0);
62

@NicoloRibaudo
Worker thread, "message" listener
3. Perform the task and send the result to the main thread
const result = await doSomethingAsync();
port.postMessage({ result });
port.close();
Atomics.store(signal, 0, 1);
Atomics.notify(signal, 0);
63

@NicoloRibaudo
Main thread, doSomethingSync
4. After waking up, read the result and return it
const { receiveMessageOnPort } = require("worker_threads");
... const { port1, port2 } = new MessageChannel(); ...
... Atomics.wait(signal, 0, 0); ...
const { result } = receiveMessageOnPort(port2);
return result;
64

@NicoloRibaudo
Production code? Error handling
65
In case of an error, we
must manually report it
to the main thread.

@NicoloRibaudo
Chapter 4:
When reality hits hard
— running Babel's tests as native ESM
66

@NicoloRibaudo
Running Babel's tests as native ESM
Problem #1
Problem #2
67
ESM compiled to CommonJS behaves diﬀerently
from ESM that runs natively in Node.js
Our test runner, Jest, didn't properly
support running ESM

@NicoloRibaudo
The __esModule convention
68
// src/main.js
import circ from "./math.js";
circ(2); // 12.56
// src/math.js
export const PI = 3.14;
export default function (r) {
return 2 * PI * r;
}
// dist/main.js
var _math = require("./math.js");
_math.default(2); // 12.56
// dist/math.js
const PI = 3.14; exports.PI = PI;
function _default(r) {
return 2 * PI * r;
}
exports.default = _default;

@NicoloRibaudo
69
// src/main.js
import circ from "../libs/math.cjs";
circ(2); // 12.56
// src/math.js
return 2 * PI * r;
}
// dist/main.js
var _math = require("../libs/math.cjs");
_math(2); // 12.56
// dist/math.js
return 2 * PI * r;
}
// libs/math.cjs
module.exports = function (r) {
return 2 * PI * r;
};
No more .default!

@NicoloRibaudo
70
// src/main.js
import circ from "../libs/math.cjs";
circ(2); // 12.56
// src/math.js
return 2 * PI * r;
}
// dist/main.js
var _math = {
default: require("../libs/math.cjs"),
};
// dist/math.js
return 2 * PI * r;
}
// libs/math.cjs
module.exports = function (r) {
return 2 * PI * r;
};

@NicoloRibaudo
71
// src/main.js
circ(2); // 12.56
// dist/main.js
var _math = _interopRequireDefault(
require("./math.js")
);
function _interopRequireDefault(obj) {
return obj is ESM compiled to CommonJS ? obj : { default: obj };
}

@NicoloRibaudo
72
// src/math.js
return 2 * PI * r;
}
// dist/math.js
return 2 * PI * r;
}
function _interopRequireDefault(obj) {
return obj && obj.__esModule ? obj : { default: obj };
}
"This was ESM, compiled to CJS"

@NicoloRibaudo
73

@NicoloRibaudo
74
● Babel
● Traceur
● tsc (TypeScript)
● SWC
● …
● Webpack
● Rollup (@rollup/plugin-commonjs
)
● Parcel
● Vite
● ESBuild
● …
It's supported by every JavaScript transpiler and bundler:

@NicoloRibaudo
75
It's supported by every JavaScript transpiler and bundler.
It's not supported by Node.js:
// main.js
import circ from "./math.cjs";
console.log(circ);
// math.cjs
return 2 * PI * r;
}
Always logs
{ __esModule: true,
PI: 3.14,
default: [function] }

@NicoloRibaudo
Converting some ﬁles to ESM
76
// math.js
return 2 * PI * r;
}
// test.js
assert(circ(2) === 12.56);
// math.js
exports.default = function (r) {
return 2 * PI * r; };
// test.js
require("./math.js") );
assert(_math.default(2) === 12.56);
✅
✅

@NicoloRibaudo
// test.js
assert(circ(2) === 12.56);
77
// math.js
return 2 * PI * r; }; // test.js
assert(_math.default(2) === 12.56);
❌
✅

@NicoloRibaudo
// test.js
assert(circ.default(2) === 12.56);
78
// math.js
assert(_math.default.default(2) === 12.56);
✅
❌

@NicoloRibaudo
// test.js
assert(circ.default(2) === 12.56);
79
// math.js
var _math = {
default: require("./math.js") };
assert(_math.default.default(2) === 12.56);
✅
✅

@NicoloRibaudo
importInterop: "node"
"This import should be compiled to match Node.js' behavior,
without checking the __esModule ﬂag."
80

@NicoloRibaudo
"This import should be compiled to match Node.js' behavior,
without checking the __esModule ﬂag."
81
// test/index.js
// Standard __esModule interop
import helper from "./helper.js";
// importInterop: "node"
import dep from "dep";
["@babel/transform-modules-commonjs", {
importInterop(source) {
if (source.startsWith(".")) {
return "babel";
}
return "node";
}
}]
babel-core
|- test
| |- index.js
| - helper.js
- node_modules
- dep
- index.js

@NicoloRibaudo
82

@NicoloRibaudo
Problem #1
Problem #2
83
support running ESM
Solved ✓

@NicoloRibaudo
Jest support for native ESM
84

@NicoloRibaudo
Jest runs every test in a virtualized context, using Node.js' vm module, to:
● isolate every test file, so that failing or misbehaving tests don't affect
other tests
● abstract and control the linking process between modules, to intercept all
requires/imports and:
○ allow mocking dependencies
○ transpile modules on-the-fly
85

@NicoloRibaudo
Node.js support for ESM in virtual vm contexts is still… rough.
86

@NicoloRibaudo
jest-light-runner
Jest supports implementing custom test runners,
to deﬁne how to execute a test.
87
https://github.com/nicolo-ribaudo/jest-light-runner

@NicoloRibaudo
jest-light-runner
88
Blazing fast!

@NicoloRibaudo
Problem #1
Problem #2
89
support running ESM
Solved ✓
Solved ✓

@NicoloRibaudo
Chapter 5:
Getting ready for an
ESM release
90

@NicoloRibaudo
Dual packages
91
Node.js supports packages with multiple implementations, that are
conditionally required/imported.
// package.json
{
"name": "your-package",
"exports": {
"import": "./esm-dist/index.mjs",
"require": "./cjs-dist/index.js"
}
}

@NicoloRibaudo
Dual packages
Converting to a "dual package" while preserving compatibility with all the
Node.js versions and various tools is incredibly complex.
92

@NicoloRibaudo
Without breaking changes
Dual packages
93

@NicoloRibaudo
Dual packages
Very high risk of breaking changes: the complete migration
is deferred to the next major release (Babel 8)
94

@NicoloRibaudo
Dual packages development
● During development, wether it's ESM or ESM-compiled-to-CJS should just
be a compilation detail
● Our codebase should always be valid in both modes
make use-cjs make use-esm
● We test both ESM and ESM-compiled-to-CJS on CI
● We are always ready to publish an ESM release, by simply ﬂipping a ﬂag
95

@NicoloRibaudo
Maximizing backwards compatibility
"ESM cannot be synchronously imported from CommonJS in Node.js"
~ me, many slides ago
96
const babel = require("@babel/core");
const fs = require("fs/promises");
babel.transformAsync(inputCode)
.then(({ code }) =>
fs.writeFile("./src/output.js", code)
);
const { types: t } = require("@babel/core");
module.exports = function myPlugin() {
return {
visitor: {
NumericLiteral(path) {
path.replaceWith(
t.stringLiteral("foo"),
);
} } };
};
How do we preserve compatibility with existing CommonJS Babel usages?

@NicoloRibaudo
Babel consumers
1. require() Babel
2. Call one of the Babel API entry points,
such as transformSync,
transformAsync, parseAsync, etc.
97
Babel plugins
1. Babel is loaded by someone else
2. Babel loads the plugin
3. The plugin require()s Babel and uses
its utilities
const {
types: t,
template,
} = require("@babel/core");
Maximizing backwards compatibility

@NicoloRibaudo
Instead of duplicating the implementation in CommonJS and ESM ﬁles,
CommonJS can act as a "proxy" over the ESM implementation.
There must still be an asynchronous step somewhere, but for libraries that
already oﬀered an async API this should be good enough.
98
CommonJS proxies

@NicoloRibaudo
Babel consumers
1. require() Babel
2. Call one of the Babel API entry points,
such as transformSync,
transformAsync, parseAsync, etc.
99
CommonJS proxies
// @babel/core/index.mjs
export async function transformAsync() {
/* ... */
}
export function transformSync() {
/* ... */
}
// @babel/core/index.cjs
let babel;
exports.transformAsync = async function () {
babel ??= await import("@babel/core");
return babel.transformAsync();
};
exports.transformSync = function () {
if (!babel) throw new Error("Not loaded yet");
return babel.transformSync();
};

@NicoloRibaudo
Babel plugins
1. Babel is loaded by someone else
2. Babel loads the plugin
3. The plugin require()s Babel and uses
its utilities
const {
types: t,
template,
} = require("@babel/core");
100
CommonJS proxies
// @babel/core/index.mjs
import { createRequire } from "module";
const require = createRequire(import.meta.url);
const cjsProxy = require("./index.cjs");
import * as thisFile from "./index.mjs";
cjsProxy.__initialize(thisFile);
// @babel/core/index.cjs
let babel;
exports.transformAsync = function () { /*..*/ };
exports.__initialize = function (b) {
babel = b;
exports.types = b.types;
exports.template = b.template;
/* ... */
};

@NicoloRibaudo
One more thing!
102

@NicoloRibaudo 103
@nicolo-ribaudo @liuxingbaoyu
@JLHwung
Babel's development is entirely funded by donations.
If you rely on Babel at work, talk to your company to get them to
sponsor the project!
One more thing!
https://opencollective.com/babel
Need help talking to your company? team@babeljs.io

Migrating Babel from CommonJS to ESM

Recommended

Recommended

More Related Content

Similar to Migrating Babel from CommonJS to ESM

Similar to Migrating Babel from CommonJS to ESM (20)

More from Igalia

More from Igalia (20)

Recently uploaded

Recently uploaded (20)

Migrating Babel from CommonJS to ESM