Common scenarios in vcl

Common
scenarios
in VCL
By Thijs Feryn

This is my first
time
in København

✓Only GET & HEAD
✓No authorization headers
✓No cookies
✓No set-cookies
✓Valid cache-control/expires
headers
When does Varnish
cache? Some rules …

Lotsof
developers
don’tfollow
thoserules

Out of the box
These are the main reasons
why Varnish will not work

Varnish
Configuration
Language

Scriptingthe
Varnish
behaviour

VCL
Recv
VCL
Hash
VCL
Miss
VCL
Hit
VCL
Backend
Response
VCL
Deliver
VCL
Purge
VCL
Error
VCL
Pipe
VCL
Pass
VCL
Synth
VCL
Backend
Error

✓vcl_recv: receive request
✓vcl_hash: compose cache key
✓vcl_miss: not found in cache
✓vcl_hit: found in cache
✓vcl_pass: don’t store in cache
✓vcl_pipe: bypass cache
✓vcl_backend_fetch: connect to
backend
✓vcl_backend_response: response
from backend
✓vcl_backend_error: backend fetch
failed

✓vcl_purge: after successful purge
✓vcl_synth: send synthetic output
✓vcl_deliver: return data to client
✓vcl_init: initialize VMODs
✓vcl_fini: discard VMODs

vcl 4.0;
backend default {
.host = "127.0.0.1";
.port = "8080";
}
Minimal VCL

vcl 4.0;
backend default {
.host = "127.0.0.1";
.port = "80";
.max_connections = 300;
.first_byte_timeout = 300s;
.connect_timeout = 5s;
.between_bytes_timeout = 2s;
}
More backend work

vcl 4.0;
backend default {
.host = "127.0.0.1";
.port = "80";
.probe = {
.url = "/";
.interval = 5s;
.timeout = 1s;
.window = 5;
.threshold = 3;
}
}
Even more backend work

vcl 4.0;
backend default {
.host = "127.0.0.1";
.port = "80";
.probe = {
.request =
"HEAD / HTTP/1.1"
"Host: localhost"
"Connection: close"
"User-Agent: Varnish Health Probe";
.interval = 5s;
.timeout = 1s;
.window = 5;
.threshold = 3;
}
}
Even more backend work

vcl 4.0;
import directors;
backend server1 {
.host = “1.2.3.4”;
.port = "80";
.probe = {
.request =
"HEAD / HTTP/1.1"
"Host: localhost"
"Connection: close"
.interval = 5s;
.timeout = 1s;
.window = 5;
.threshold = 3;
}
}
Loadbalancing: server 1

backend server2 {
.host = “1.2.3.5”;
.port = "80";
.probe = {
.request =
"HEAD / HTTP/1.1"
"Host: localhost"
"Connection: close"
.interval = 5s;
.timeout = 1s;
.window = 5;
.threshold = 3;
}
}
Loadbalancing: server 2

sub vcl_init {
new vdir = directors.round_robin();
vdir.add_backend(server1);
}
sub vcl_recv {
set req.backend_hint = vdir.backend();
}
Loadbalancing: round robin

sub vcl_init {
new vdir = directors.random();
vdir.add_backend(server1,2);
vdir.add_backend(server2,3);
}
sub vcl_recv {
}
Loadbalancing: random

sub vcl_init {
new vdir = directors.fallback();
}
sub vcl_recv {
}
Loadbalancing: fallback

sub vcl_init {
new vdir = directors.hash();
}
sub vcl_recv {
set req.backend_hint = vdir.backend(req.url);
}
Loadbalancing: URL hash

sub vcl_init {
new vdir = directors.hash();
}
sub vcl_recv {
set req.backend_hint = vdir.backend(client.identity);
}
Loadbalancing: IP hash

sub vcl_recv {
if(req.url ~ “^/products”) {
set req.backend_hint = server1;
} else {
set req.backend_hint = server2;
}
}
Conditional loadbalacning

vcl 4.0;
import std;
sub vcl_recv {
set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");
set req.url = std.querysort(req.url);
if (req.url ~ "#") {
set req.url = regsub(req.url, "#.*$", "");
}
if (req.url ~ "?$") {
set req.url = regsub(req.url, "?$", "");
}
if (req.restarts == 0) {
if (req.http.Accept-Encoding) {
if (req.http.User-Agent ~ "MSIE 6") {
unset req.http.Accept-Encoding;
} elsif (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";
} elsif (req.http.Accept-Encoding ~ "deflate") {
set req.http.Accept-Encoding = "deflate";
} else {
unset req.http.Accept-Encoding;
}
}
}
}
Normalize

vcl 4.0;
sub vcl_recv {
if (req.url ~ "^[^?]*.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|
flv|gif|gz|ico|jpeg|jpg|js|less|mka|mkv|mov|mp3|mp4|mpeg|mpg|odt|otf|
ogg|ogm|opus|pdf|png|ppt|pptx|rar|rtf|svg|svgz|swf|tar|tbz|tgz|ttf|
txt|txz|wav|webm|webp|woff|woff2|xls|xlsx|xml|xz|zip)(?.*)?$") {
unset req.http.Cookie;
return (hash);
}
}
sub vcl_backend_response {
if (bereq.url ~ "^[^?]*.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|
unset beresp.http.set-cookie;
}
if (bereq.url ~ "^[^?]*.(7z|avi|bz2|flac|flv|gz|mka|mkv|mov|mp3|mp4|
mpeg|mpg|ogg|ogm|opus|rar|tar|tgz|tbz|txz|wav|webm|xz|zip)(?.*)?$") {
unset beresp.http.set-cookie;
set beresp.do_stream = true;
set beresp.do_gzip = false;
}
}
Cache static assets

Doyoureally
wanttocache
staticassets?

Nginxor
Apachecan
befast
enoughfor
that

vcl 4.0;
import std;
sub vcl_recv {
if (req.url ~ "^[^?]*.(7z|avi|bmp|bz2|css|csv|doc|docx|eot|flac|
unset req.http.Cookie;
return (pass);
}
}
Don’t cache static assets

sub vcl_recv {
if (req.url ~ "^/status.php$" ||
req.url ~ "^/update.php$" ||
req.url ~ "^/admin$" ||
req.url ~ "^/admin/.*$" ||
req.url ~ "^/user$" ||
req.url ~ "^/user/.*$" ||
req.url ~ "^/flag/.*$" ||
req.url ~ "^.*/ajax/.*$" ||
req.url ~ "^.*/ahah/.*$") {
return (pass);
}
}
URL blacklist

sub vcl_recv {
if (req.url ~ "^/products/?"
return (hash);
}
}
URL whitelist

vcl 4.0;
sub vcl_recv {
set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "_ga=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "_gat=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "utmctr=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "utmcmd.=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "utmccn.=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "__gads=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "__qc.=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "__atuv.=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "^;s*", "");
if (req.http.cookie ~ "^s*$") {
unset req.http.cookie;
}
}
Remove tracking cookies

vcl 4.0;
sub vcl_recv {
if (req.http.Cookie) {
set req.http.Cookie = ";" + req.http.Cookie;
set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
set req.http.Cookie = regsuball(req.http.Cookie, ";(PHPSESSID)=", "; 1=");
set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");
}
}
}
Only keep session cookie

sub vcl_recv {
if (req.http.Cookie) {
set req.http.Cookie = ";" + req.http.Cookie;
set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
set req.http.Cookie = regsuball(req.http.Cookie, ";(language)=", "; 1=");
set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");
set req.http.Cookie = regsuball(req.http.Cookie, "^[; ]+|[; ]+$", "");
return(pass);
}
return(hash);
}
}
sub vcl_hash {
hash_data(regsub( req.http.Cookie, "^.*language=([^;]*);*.*$", "1" ));
}
Language cookie cache variation

Alternative
language
cache
variation

sub vcl_hash {
hash_data(req.http.Accept-Language);
}
Language cookie cache variation
Or just send a
“Vary:Accept-Language”
header

sub vcl_hash {
hash_data(req.http.Cookie);
}
Hash all cookies

sub vcl_recv {
set req.http.Surrogate-Capability = "key=ESI/1.0";
}
if (beresp.http.Surrogate-Control ~ "ESI/1.0") {
unset beresp.http.Surrogate-Control;
set beresp.do_esi = true;
}
}
Edge Side Includes

<?php
header("Cache-Control: public,must-revalidate,s-maxage=10");
echo "Date in the ESI tag: ".date('Y-m-d H:i:s').'<br />';
<?php
header("Cache-Control: no-store");
header(“Surrogate-Control: content='ESI/1.0'");
echo '<esi:include src="/esi.php" />'.PHP_EOL;
echo "Date in the main page: ".date('Y-m-d H:i:s').'<br />';
Main page
ESI frame:
esi.php
Cached for
10 seconds
Not cached

set beresp.ttl = 3h;
}
Control Time To Live

if (beresp.ttl <= 0s || beresp.http.Set-Cookie ||
beresp.http.Vary == "*") {
set beresp.ttl = 120s;
set beresp.uncacheable = true;
return (deliver);
}
}
Control Time To Live

sub vcl_deliver {
if (obj.hits > 0) {
set resp.http.X-Cache = "HIT";
} else {
set resp.http.X-Cache = "MISS";
}
}
Debugging

sub vcl_deliver {
unset resp.http.X-Powered-By;
unset resp.http.Server;
unset resp.http.X-Drupal-Cache;
unset resp.http.X-Varnish;
unset resp.http.Via;
unset resp.http.Link;
unset resp.http.X-Generator;
}
Anonymize

acl purge {
"localhost";
"127.0.0.1";
"::1";
}
sub vcl_recv {
if (req.method == "PURGE") {
if (!client.ip ~ purge) {
return (synth(405, “Not allowed.”));
}
return (purge);
}
}
Purging

acl purge {
"localhost";
"127.0.0.1";
"::1";
}
set beresp.http.x-url = bereq.url;
set beresp.http.x-host = bereq.http.host;
}
sub vcl_deliver {
unset resp.http.x-url;
unset resp.http.x-host;
}
sub vcl_recv {
if (req.method == "PURGE") {
if (!client.ip ~ purge) {
return (synth(405, "Not allowed"));
}
if(req.http.x-purge-regex) {
ban("obj.http.x-host == " + req.http.host + " && obj.http.x-url ~ " +
req.http.x-purge-regex);
} else {
ban("obj.http.x-host == " + req.http.host + " && obj.http.x-url == " +
req.url);
}
return (synth(200, "Purged"));
}
}
Banning

Banning
curl -XPURGE -H "x-purge-regex:/products"
"http://example.com"
curl -XPURGE "http://example.com/products"

set beresp.grace = 6h;
}
Grace mode

Common scenarios in vcl

More Related Content

What's hot

Similar to Common scenarios in vcl

More from Varnish Software

Recently uploaded

Common scenarios in vcl