Node.js: scalability tips - CityJS 2020

Node.js Scalability Tips
Luciano Mammino ( )@loige
loige.link/node-scale-city
2020-09-15 1

👋 Hello, I am Luciano!
Principal Software Engineer at FabFitFun
Blog:
Twitter:
GitHub:
loige.co
@loige
@lmammino
nodejsdp.link/buy
2

👋 Hello, I am Luciano!
Principal Software Engineer at FabFitFun
Blog:
Twitter:
GitHub:
loige.co
@loige
@lmammino
nodejsdp.link/buy
CITYJS SPECIAL DISCOUNT: 20% OFF on Amazon.com - www.amazon.com/gp/mpc/A2L4T1QGT3DXYX 2

@loige
Get the slides!
3

Node.js + Scalability? 🤔
@loige
4

"Scalability is the property of a system to
handle a growing amount of work by
adding resources to the system"
— Wikipedia
@loige
8

"A service is said to be scalable if when
we increase the resources in a
system, it results in increased
performance in a manner
proportional to resources added"
— Werner Vogels
@loige
9

🛫
Tip 1.
Establish a baseline
@loige
10

/?data=hello%20cityjs
https://repl.it/@lmammino/QRGen
@loige
11

const { createServer } = require('http')
const { URL } = require('url')
const QRCode = require('qrcode')
createServer(function handler (req, res) {
const url = new URL(req.url, 'http://localhost:8080')
const data = url.searchParams.get('data')
if (!data) {
res.writeHead(400) // bad request
return res.end()
}
res.writeHead(200, { 'Content-Type': 'image/png' })
QRCode.toFileStream(res, data, { width: 300 })
})
.listen(8080)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
@loige
12

if (!data) {
return res.end()
}
})
.listen(8080)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
1
2
3
4
createServer(function handler (req, res) {5
const url = new URL(req.url, 'http://localhost:8080')6
const data = url.searchParams.get('data')7
if (!data) {8
res.writeHead(400) // bad request9
return res.end()10
}11
res.writeHead(200, { 'Content-Type': 'image/png' })12
QRCode.toFileStream(res, data, { width: 300 })13
})14
.listen(8080)15
@loige
12

if (!data) {
return res.end()
}
})
.listen(8080)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
1
2
3
4
if (!data) {8
return res.end()10
}11
})14
.listen(8080)15
})
.listen(8080)
const { createServer } = require('http')1
const { URL } = require('url')2
const QRCode = require('qrcode')3
4
5
if (!data) {8
return res.end()10
}11
14
15
@loige
12

if (!data) {
return res.end()
}
})
.listen(8080)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
1
2
3
4
if (!data) {8
return res.end()10
}11
})14
.listen(8080)15
})
.listen(8080)
4
5
if (!data) {8
return res.end()10
}11
14
15
4
6
7
if (!data) {8
return res.end()10
}11
})14
.listen(8080)15
@loige
12

if (!data) {
return res.end()
}
})
.listen(8080)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
1
2
3
4
if (!data) {8
return res.end()10
}11
})14
.listen(8080)15
})
.listen(8080)
4
5
if (!data) {8
return res.end()10
}11
14
15
4
6
7
if (!data) {8
return res.end()10
}11
})14
.listen(8080)15
if (!data) {
return res.end()
}
4
8
9
10
11
})14
.listen(8080)15
@loige
12

if (!data) {
return res.end()
}
})
.listen(8080)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
1
2
3
4
if (!data) {8
return res.end()10
}11
})14
.listen(8080)15
})
.listen(8080)
4
5
if (!data) {8
return res.end()10
}11
14
15
4
6
7
if (!data) {8
return res.end()10
}11
})14
.listen(8080)15
if (!data) {
return res.end()
}
4
8
9
10
11
})14
.listen(8080)15
4
if (!data) {8
return res.end()10
}11
12
13
})14
.listen(8080)15
@loige
12

if (!data) {
return res.end()
}
})
.listen(8080)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
1
2
3
4
if (!data) {8
return res.end()10
}11
})14
.listen(8080)15
})
.listen(8080)
4
5
if (!data) {8
return res.end()10
}11
14
15
4
6
7
if (!data) {8
return res.end()10
}11
})14
.listen(8080)15
if (!data) {
return res.end()
}
4
8
9
10
11
})14
.listen(8080)15
4
if (!data) {8
return res.end()10
}11
12
13
})14
.listen(8080)15
if (!data) {
return res.end()
}
})
.listen(8080)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
@loige
12

autocannon -c 200 --on-port / -- node server.js
wrk
node server.js&
wrk -t8 -c200 -d10s http://localhost:8080/
@loige
13

autocannon -c 200 --on-port /?data=hello%20cityjs -- node server.js
Running 10s test @ http://localhost:8080/?data=hello%20cityjs
200 connections
┌─────────┬─────────┬─────────┬─────────┬─────────┬────────────┬─────────┬────────────┐
│ Stat │ 2.5% │ 50% │ 97.5% │ 99% │ Avg │ Stdev │ Max │
├─────────┼─────────┼─────────┼─────────┼─────────┼────────────┼─────────┼────────────┤
│ Latency │ 1899 ms │ 1951 ms │ 2053 ms │ 2054 ms │ 1964.92 ms │ 99.9 ms │ 3364.03 ms │
└─────────┴─────────┴─────────┴─────────┴─────────┴────────────┴─────────┴────────────┘
┌───────────┬─────┬──────┬─────────┬────────┬────────┬────────┬─────────┐
│ Stat │ 1% │ 2.5% │ 50% │ 97.5% │ Avg │ Stdev │ Min │
├───────────┼─────┼──────┼─────────┼────────┼────────┼────────┼─────────┤
│ Req/Sec │ 0 │ 0 │ 30 │ 199 │ 99.5 │ 94.27 │ 30 │
├───────────┼─────┼──────┼─────────┼────────┼────────┼────────┼─────────┤
│ Bytes/Sec │ 0 B │ 0 B │ 50.7 kB │ 336 kB │ 168 kB │ 159 kB │ 50.7 kB │
└───────────┴─────┴──────┴─────────┴────────┴────────┴────────┴─────────┘
Req/Bytes counts sampled once per second.
995 requests in 10.08s, 1.68 MB read
@loige
14

⛅
Tip 1-bis
Also, find out your ceiling
@loige
15

createServer((req, res) => {
if (req.method === 'GET' && req.url === '/') {
res.writeHead(200, { 'Content-Type': 'text/plain' })
res.end('Hello Worldn')
} else {
res.statusCode = 404
res.end()
}
})
.listen(8080)
1
2
3
4
5
6
7
8
9
10
11
12
@loige
16

createServer((req, res) => {
if (req.method === 'GET' && req.url === '/') {
} else {
res.end()
}
})
.listen(8080)
1
2
3
4
5
6
7
8
9
10
11
12
2
createServer((req, res) => {3
if (req.method === 'GET' && req.url === '/') {4
5
6
} else {7
res.end()9
}10
})11
.listen(8080)12
@loige
16

autocannon -c 200 --on-port / -- node server.js
Running 10s test @ http://localhost:8080/
200 connections
┌─────────┬──────┬──────┬───────┬───────┬─────────┬─────────┬──────────┐
│ Stat │ 2.5% │ 50% │ 97.5% │ 99% │ Avg │ Stdev │ Max │
├─────────┼──────┼──────┼───────┼───────┼─────────┼─────────┼──────────┤
│ Latency │ 3 ms │ 5 ms │ 11 ms │ 14 ms │ 5.51 ms │ 2.71 ms │ 80.63 ms │
└─────────┴──────┴──────┴───────┴───────┴─────────┴─────────┴──────────┘
┌───────────┬─────────┬─────────┬────────┬─────────┬─────────┬─────────┬─────────┐
│ Stat │ 1% │ 2.5% │ 50% │ 97.5% │ Avg │ Stdev │ Min │
├───────────┼─────────┼─────────┼────────┼─────────┼─────────┼─────────┼─────────┤
│ Req/Sec │ 21087 │ 21087 │ 34623 │ 35487 │ 33258.4 │ 4107.01 │ 21077 │
├───────────┼─────────┼─────────┼────────┼─────────┼─────────┼─────────┼─────────┤
│ Bytes/Sec │ 3.29 MB │ 3.29 MB │ 5.4 MB │ 5.54 MB │ 5.19 MB │ 641 kB │ 3.29 MB │
└───────────┴─────────┴─────────┴────────┴─────────┴─────────┴─────────┴─────────┘
Req/Bytes counts sampled once per second.
333k requests in 10.1s, 51.9 MB read
@loige
17

🍾
Tip 2.
Find your bottleneck
@loige
18

Clinic.js
clinicjs.org
@loige
19

clinic doctor --autocannon [ -c 200 '/?data=hello%20cityjs' ] -- node server.js
@loige20

clinic flame --autocannon [ -c 200 '/?data=hello%20cityjs' ] -- node server.js
@loige21

clinic bubble --autocannon [ -c 200 '/?data=hello%20cityjs' ] -- node server.js
@loige22

🎳
Tip 3.
Understand your goals
@loige
23

What do we optimize for?
Throughput?
Memory?
Latency?
@loige
24

👁
Tip 4.
Always "observe"
@loige
25

I mean, in production!
Logs - Metrics - Traces
@loige
26

🚀
Tip 5.
Scale your architecture
@loige
27

Performance != Scalability
@loige
28

How can we scale a system
by adding resources?
@loige
29

The " "Scale Cube
x-axis
cloning
z-axis
partitioning
y-axis
functional
decomposition
@loige
30

Cloning
Reverse proxy
31
Inside the same server
Load Balancer
Using multiple server
@loige

The modulecluster
Master process
Worker
process
Worker
process
Worker
process
32
@loige

const cluster = require('cluster')
const numCPUs = require('os').cpus().length
if (cluster.isMaster) {
// Fork workers
for (let i = 0; i < numCPUs; i++) {
cluster.fork()
}
} else {
// Worker code
require('./server.js')
}
1
2
3
4
5
6
7
8
9
10
11
12
@loige
33

// Fork workers
cluster.fork()
}
} else {
// Worker code
}
1
2
3
4
5
6
7
8
9
10
11
12 3-4x req/sec
(8 core)
@loige
33

// Fork workers
cluster.fork()
}
} else {
// Worker code
}
1
2
3
4
5
6
7
8
9
10
11
12
const cluster = require('cluster')1
const numCPUs = require('os').cpus().length2
3
if (cluster.isMaster) {4
// Fork workers5
for (let i = 0; i < numCPUs; i++) {6
cluster.fork()7
}8
} else {9
// Worker code10
require('./server.js')11
}12 3-4x req/sec
(8 core)
@loige
33

// Fork workers
cluster.fork()
}
} else {
// Worker code
}
1
2
3
4
5
6
7
8
9
10
11
12
3
// Fork workers5
cluster.fork()7
}8
} else {9
// Worker code10
}12
2
3
// Fork workers5
cluster.fork()7
}8
} else {9
// Worker code10
}12 3-4x req/sec
(8 core)
@loige
33

// Fork workers
cluster.fork()
}
} else {
// Worker code
}
1
2
3
4
5
6
7
8
9
10
11
12
3
// Fork workers5
cluster.fork()7
}8
} else {9
// Worker code10
}12
2
3
// Fork workers5
cluster.fork()7
}8
} else {9
// Worker code10
}12
} else {
}
3
4
// Fork workers5
cluster.fork()7
}8
9
// Worker code10
12 3-4x req/sec
(8 core)
@loige
33

// Fork workers
cluster.fork()
}
} else {
// Worker code
}
1
2
3
4
5
6
7
8
9
10
11
12
3
// Fork workers5
cluster.fork()7
}8
} else {9
// Worker code10
}12
2
3
// Fork workers5
cluster.fork()7
}8
} else {9
// Worker code10
}12
} else {
}
3
4
// Fork workers5
cluster.fork()7
}8
9
// Worker code10
12
// Fork workers
cluster.fork()
}
3
5
6
7
8
} else {9
// Worker code10
}12 3-4x req/sec
(8 core)
@loige
33

// Fork workers
cluster.fork()
}
} else {
// Worker code
}
1
2
3
4
5
6
7
8
9
10
11
12
3
// Fork workers5
cluster.fork()7
}8
} else {9
// Worker code10
}12
2
3
// Fork workers5
cluster.fork()7
}8
} else {9
// Worker code10
}12
} else {
}
3
4
// Fork workers5
cluster.fork()7
}8
9
// Worker code10
12
// Fork workers
cluster.fork()
}
3
5
6
7
8
} else {9
// Worker code10
}12
// Worker code
3
// Fork workers5
cluster.fork()7
}8
} else {9
10
11
}12 3-4x req/sec
(8 core)
@loige
33

// Fork workers
cluster.fork()
}
} else {
// Worker code
}
1
2
3
4
5
6
7
8
9
10
11
12
3
// Fork workers5
cluster.fork()7
}8
} else {9
// Worker code10
}12
2
3
// Fork workers5
cluster.fork()7
}8
} else {9
// Worker code10
}12
} else {
}
3
4
// Fork workers5
cluster.fork()7
}8
9
// Worker code10
12
// Fork workers
cluster.fork()
}
3
5
6
7
8
} else {9
// Worker code10
}12
// Worker code
3
// Fork workers5
cluster.fork()7
}8
} else {9
10
11
}12
// Fork workers
cluster.fork()
}
} else {
// Worker code
}
1
2
3
4
5
6
7
8
9
10
11
12 3-4x req/sec
(8 core)
@loige
33

You could also use
Check out !
Worker Threads
piscina
@loige
34

Cloning is the easiest strategy to
scale a service...

... as long as your application is
"Stateless"
@loige
35

API Gateway
Functional decomposition
a.k.a. "Micro-services"
36
/products
/cart
cart DB
products
DB
@loige

API Gateway
Functional decomposition
a.k.a. "Micro-services"
37
/products
/cart
Functional decomposition can
also be combined with cloning!
cart DB
products
DB
@loige

Node.js is great for microservices
@loige
38

Microservices can also help with
scaling the organisation!
@loige
39

Microservices add complexity
Observability
Deployments
Versioning
Integration
@loige
40

Partitioning
Service and Data Partitioning along Customer Boundaries
Shard partitioning
/products/[A-L]/
/products/[M-Z]/
DB 2
41
DB 1
@loige

Partitioning is generally used
to scale databases
and
SaaS software geographically
@loige
42

Summary
🛫 Establish a baseline
@loige
43

Summary
🍾 Find your bottleneck
@loige
43

Summary
🎳 Understand your goals
@loige
43

Summary
👁 Always "observe"
@loige
43

Summary
👁 Always "observe"
🚀 Scale your architecture
(cloning, decomposition & partitioning)
@loige
43

Thank you!
Special thanks to ,
, ,
, ,
, ,
, ,
, , ,
, , ,
, , ,
, , ,
Icons and SVGs by
@StefanoAbalsamo
@matteocollina @dagonzago
@NullishCoalesce @DublinSvelte
@KViglucci @gjohnson391
@lucamaraschi @laurekamalandua
@giltayar @mrm8488 @adrirai
@haraﬁse @EugeneWare @Jauny
@tlivings @michaelcﬁne @leojino
@shahidontech @Lordoomer @zsadigov
@dottorblaster
freepik.com
@loige
44

Node.js: scalability tips - CityJS 2020

Recommended

Recommended

More Related Content

More from Luciano Mammino

More from Luciano Mammino (20)

Node.js: scalability tips - CityJS 2020