Networks and APIs

// A pretend server: just a function from a request to a response.
// It runs here, in this tab. Nothing goes over a network.
const books = {
  42: { id: 42, title: "Dune", year: 1965 },
};
​
function server(request) {
  console.log("server got:", request.method, request.path);
  const match = request.path.match(/^\/books\/(\d+)$/);
  if (request.method === "GET" && match && books[match[1]]) {
    return { status: 200, body: books[match[1]] };
  }
  return { status: 404, body: { error: "No such book" } };
}
​
// The client asks; the server answers. That is the whole pattern.
const response = server({ method: "GET", path: "/books/42" });
console.log("client got:", response.status, response.body);
​
const missing = server({ method: "GET", path: "/books/7" });
console.log("client got:", missing.status, missing.body);

// HTTP/1.1 is text. This is a real request, character for character.
const body = JSON.stringify({ title: "Dune", year: 1965 });
const request = [
  "POST /books HTTP/1.1",            // method, target, version
  "Host: api.example.com",           // headers, one per line
  "Content-Type: application/json",
  "Content-Length: " + new TextEncoder().encode(body).length,
  "",                                // an empty line ends the headers
  body,                              // then the body
].join("\r\n");
​
// The server's first job: parse that text back into parts.
function parseRequest(text) {
  const cut = text.indexOf("\r\n\r\n");
  const head = text.slice(0, cut);
  const [startLine, ...headerLines] = head.split("\r\n");
  const [method, target, version] = startLine.split(" ");
  const headers = {};
  for (const line of headerLines) {
    const colon = line.indexOf(":");
    headers[line.slice(0, colon).toLowerCase()] = line.slice(colon + 1).trim();
  }
  return { method, target, version, headers, body: text.slice(cut + 4) };
}
​
console.log(request);
parseRequest(request);

// fakeFetch has the same shape as the browser's fetch(), but the server
// is the function handle() below, in this tab. No network at all.
async function handle(req) {
  const url = new URL(req.url);
  if (req.method === "GET" && url.pathname === "/books/42") {
    return Response.json({ id: 42, title: "Dune" });
  }
  if (req.method === "POST" && url.pathname === "/books") {
    const book = await req.json();
    return Response.json({ id: 43, ...book }, {
      status: 201,
      headers: { Location: "/books/43" },
    });
  }
  return Response.json({ error: "not found" }, { status: 404 });
}
async function fakeFetch(path, init) {
  const req = new Request("https://api.example.com" + path, init);
  await new Promise((r) => setTimeout(r, 20));   // pretend the trip takes time
  return handle(req);
}
​
const res = await fakeFetch("/books/42");
console.log(res.status, await res.json());
​
const created = await fakeFetch("/books", {
  method: "POST",
  headers: { "Content-Type": "application/json" },
  body: JSON.stringify({ title: "Neuromancer" }),
});
console.log(created.status, created.headers.get("location"), await created.json());

// fetch() only rejects when there is no answer at all. A 404 or a 500
// is still an answer, so checking res.ok is your job.
async function handle(req) {
  const path = new URL(req.url).pathname;
  if (path === "/ok") return new Response("fine", { status: 200 });
  if (path === "/gone") return new Response("no such thing", { status: 404 });
  if (path === "/boom") return new Response("server fell over", { status: 500 });
  throw new TypeError("Failed to fetch");       // the network itself failed
}
async function fakeFetch(path) {
  return handle(new Request("https://api.example.com" + path));
}
​
for (const path of ["/ok", "/gone", "/boom", "/unplugged"]) {
  try {
    const res = await fakeFetch(path);
    console.log(path, "→", res.status, "ok:", res.ok, "·", await res.text());
  } catch (err) {
    console.log(path, "→ rejected:", err.name + ": " + err.message);
  }
}

// The server labels each version of a resource with an ETag.
// The client sends it back in If-None-Match: "I have this one. Changed?"
let book = { id: 42, title: "Dune", year: 1965 };
let version = 1;
​
async function handle(req) {
  const etag = '"v' + version + '"';
  if (req.headers.get("If-None-Match") === etag) {
    return new Response(null, { status: 304, headers: { ETag: etag } });
  }
  return Response.json(book, { headers: { ETag: etag, "Cache-Control": "no-cache" } });
}
async function fakeFetch(path, init) {
  return handle(new Request("https://api.example.com" + path, init));
}
​
const first = await fakeFetch("/books/42");
for (const [name, value] of first.headers) console.log(name + ": " + value);
const tag = first.headers.get("ETAG");           // names ignore case
console.log(first.status, (await first.text()).length, "bytes of body");
​
const again = await fakeFetch("/books/42", { headers: { "If-None-Match": tag } });
console.log(again.status, (await again.text()).length, "bytes of body");
​
book.year = 1966; version++;                     // someone edits the book
const later = await fakeFetch("/books/42", { headers: { "If-None-Match": tag } });
console.log(later.status, later.headers.get("etag"), await later.json());

const book = {
  id: 42,
  title: "Dune",
  published: new Date("1965-08-01"),
  tags: ["sci-fi", "classic"],
  rating: NaN,
  notes: undefined,
};
​
const text = JSON.stringify(book);      // what actually travels: plain text
console.log(text);
​
const back = JSON.parse(text);
console.log(typeof back.published);     // the Date came back as a string
console.log("notes" in back, back.rating);
​
// A reviver can turn known fields back into real objects
const revived = JSON.parse(text, (key, value) => key === "published" ? new Date(value) : value);
console.log(revived.published.getUTCFullYear());
​
// JSON numbers have no size limit. JavaScript numbers do.
console.log(JSON.parse('{"id": 9007199254740993}').id);

// A tiny REST API for one collection, kept in memory.
const db = new Map([[1, { id: 1, title: "Dune" }], [2, { id: 2, title: "Emma" }]]);
let nextId = 3;
​
function handle(method, path, body) {
  const [, collection, rawId] = path.split("/");     // "/books/2" → ["", "books", "2"]
  if (collection !== "books") return { status: 404 };
  if (!rawId) {                                      // the collection: /books
    if (method === "GET") return { status: 200, body: [...db.values()] };
    if (method === "POST") {
      const book = { id: nextId++, title: body.title };
      db.set(book.id, book);
      return { status: 201, headers: { Location: "/books/" + book.id }, body: book };
    }
    return { status: 405, headers: { Allow: "GET, POST" } };
  }
  const id = Number(rawId);                          // one item: /books/2
  if (!db.has(id)) return { status: 404 };
  if (method === "GET") return { status: 200, body: db.get(id) };
  if (method === "PUT") { db.set(id, { ...body, id }); return { status: 200, body: db.get(id) }; }
  if (method === "PATCH") { Object.assign(db.get(id), body); return { status: 200, body: db.get(id) }; }
  if (method === "DELETE") { db.delete(id); return { status: 204 }; }
  return { status: 405, headers: { Allow: "GET, PUT, PATCH, DELETE" } };
}
​
console.log(handle("POST", "/books", { title: "Neuromancer" }));
console.log(handle("PATCH", "/books/3", { year: 1984 }));
console.log(handle("DELETE", "/books/1"));
console.log(handle("GET", "/books/1"));
console.log(handle("GET", "/books"));
console.log(handle("POST", "/books/2"));

// 10 posts, newest first. The client reads them 4 at a time.
let posts = Array.from({ length: 10 }, (_, i) => ({ id: 10 - i }));
​
// Offset paging: "skip 4, give me 4"
function byOffset(offset, limit) {
  return posts.slice(offset, offset + limit).map((p) => p.id);
}
// Cursor paging: "give me 4 older than the last one I saw"
function byCursor(after, limit) {
  const page = posts.filter((p) => after === null || p.id < after).slice(0, limit);
  const next = page.length === limit ? page[page.length - 1].id : null;
  return { ids: page.map((p) => p.id), next };
}
​
const o1 = byOffset(0, 4);
const c1 = byCursor(null, 4);
posts.unshift({ id: 11 });            // someone posts while we're reading
const o2 = byOffset(4, 4);
const c2 = byCursor(c1.next, 4);
​
console.log("offset:", o1, o2);       // look for a repeat
console.log("cursor:", c1.ids, c2.ids, "next cursor:", c2.next);

// The network loses the first reply. The client can't tell whether the
// payment happened, so it tries again. Without a key, that charges twice.
function makeServer(useKeys) {
  const charges = [];
  const seen = new Map();               // Idempotency-Key → the reply we gave
  let calls = 0;
  function pay(req) {
    calls++;
    if (useKeys && seen.has(req.key)) return seen.get(req.key);   // replay it
    charges.push(req.amount);
    const reply = { status: 201, id: "pay_" + charges.length };
    if (useKeys) seen.set(req.key, reply);
    if (calls === 1) throw new Error("timed out: the reply was lost");
    return reply;
  }
  return { pay, charges };
}
​
function clientPays(server, key) {
  for (let attempt = 1; attempt <= 3; attempt++) {
    try {
      return server.pay({ key, amount: 30 });
    } catch (err) {
      console.log("  attempt", attempt, "→", err.message);
    }
  }
}
​
const naive = makeServer(false);
console.log("no key:", clientPays(naive, "k-81f3"), "charges:", naive.charges);
const safe = makeServer(true);
console.log("with key:", clientPays(safe, "k-81f3"), "charges:", safe.charges);

// Retry politely: wait longer after each failure, and add randomness
// ("jitter") so a thousand clients don't all retry in the same instant.
function backoff(attempt, base = 100, cap = 10000) {
  const ceiling = Math.min(cap, base * 2 ** attempt);
  return Math.round(Math.random() * ceiling);     // "full jitter"
}
​
for (let attempt = 0; attempt < 8; attempt++) {
  const most = Math.min(10000, 100 * 2 ** attempt);
  console.log("attempt", attempt, "· wait 0 to", most, "ms · this time:", backoff(attempt));
}

// A signed token built the way a JWT is: header.payload.signature, each
// part base64url-encoded. Only the server knows the secret.
const secret = "server-only-secret";
const enc = new TextEncoder();
const b64url = (bytes) => btoa(String.fromCharCode(...bytes))
  .replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
const part = (obj) => b64url(enc.encode(JSON.stringify(obj)));
​
async function sign(text) {
  const key = await crypto.subtle.importKey("raw", enc.encode(secret),
    { name: "HMAC", hash: "SHA-256" }, false, ["sign"]);
  return b64url(new Uint8Array(await crypto.subtle.sign("HMAC", key, enc.encode(text))));
}
async function login(user) {            // after checking the password
  const body = part({ alg: "HS256", typ: "JWT" }) + "." +
    part({ sub: user, role: "reader", exp: 2000000000 });
  return body + "." + await sign(body);
}
async function handle(req) {            // GET /me
  const token = (req.headers.get("Authorization") || "").replace(/^Bearer /, "");
  const [head, payload, sig] = token.split(".");
  if (!sig || sig !== await sign(head + "." + payload)) {
    return new Response("invalid token", { status: 401 });
  }
  const claims = JSON.parse(atob(payload.replace(/-/g, "+").replace(/_/g, "/")));
  if (claims.exp * 1000 < Date.now()) return new Response("expired", { status: 401 });
  return Response.json({ hello: claims.sub, role: claims.role });
}
const call = (token) => handle(new Request("https://api.example.com/me",
  { headers: token ? { Authorization: "Bearer " + token } : {} }));
​
const token = await login("ada");
console.log(token);
console.log("no token:", (await call(null)).status);
console.log("token:", await (await call(token)).json());
​
// Forge it: change the role, keep the old signature
const [h, , s] = token.split(".");
const forged = h + "." + part({ sub: "ada", role: "admin", exp: 2000000000 }) + "." + s;
console.log("forged:", (await call(forged)).status);

// A token bucket: it holds up to 5 tokens and refills 1 per second.
// Each request spends one. No token, no service: 429.
function makeBucket(capacity, perSecond) {
  let tokens = capacity;
  let last = 0;
  return function take(now) {            // now: a time in ms (a fake clock)
    tokens = Math.min(capacity, tokens + ((now - last) / 1000) * perSecond);
    last = now;
    if (tokens >= 1) {
      tokens -= 1;
      return { status: 200, remaining: Math.floor(tokens) };
    }
    return { status: 429, retryAfter: Math.ceil((1 - tokens) / perSecond) };
  };
}
​
const take = makeBucket(5, 1);
for (const t of [0, 0, 0, 0, 0, 0, 0, 1500, 1600, 4000]) {
  console.log("t =", String(t).padStart(4), "ms →", take(t));
}

// An origin is scheme + host + port. All three must match.
const page = new URL("https://app.example/books");
​
for (const other of [
  "https://app.example/api/list",
  "http://app.example/api/list",
  "https://api.app.example/list",
  "https://app.example:8443/list",
]) {
  const same = new URL(other).origin === page.origin;
  console.log(new URL(other).origin.padEnd(28), same ? "same origin" : "cross-origin");
}

// THE SERVER decides which page origins may read its answers.
const allowed = ["https://app.example"];
const serverLog = [];
​
function server(req) {
  serverLog.push(req.method + " " + req.path + " from " + req.headers.Origin);
  const cors = allowed.includes(req.headers.Origin) ? {
    "Access-Control-Allow-Origin": req.headers.Origin,
    "Access-Control-Allow-Methods": "GET, POST, DELETE",
    "Vary": "Origin",
  } : {};
  if (req.method === "OPTIONS") return { status: 204, headers: cors };  // preflight
  return { status: 200, headers: cors, body: "your reading list" };
}
​
// THE BROWSER enforces the rule, on the user's behalf.
function browserFetch(pageOrigin, method, path) {
  const simple = ["GET", "HEAD", "POST"].includes(method);   // (simplified)
  if (!simple) {
    const pre = server({ method: "OPTIONS", path, headers: { Origin: pageOrigin } });
    const ok = pre.headers["Access-Control-Allow-Origin"] === pageOrigin &&
      (pre.headers["Access-Control-Allow-Methods"] || "").split(", ").includes(method);
    if (!ok) return "blocked: preflight refused, real request never sent";
  }
  const res = server({ method, path, headers: { Origin: pageOrigin } });
  const acao = res.headers["Access-Control-Allow-Origin"];
  if (acao !== pageOrigin && acao !== "*") return "blocked: the server answered, the page may not read it";
  return res.status + " " + (res.body || "");
}
​
console.log(browserFetch("https://app.example", "GET", "/list"));
console.log(browserFetch("https://evil.example", "GET", "/list"));
console.log(browserFetch("https://evil.example", "DELETE", "/list"));
console.log(serverLog);

// A WebSocket starts life as an ordinary HTTP request:
//   GET /chat HTTP/1.1
//   Upgrade: websocket
//   Connection: Upgrade
//   Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==
// The server proves it understood by hashing the key with a fixed GUID.
const GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11";
​
async function acceptFor(key) {
  const hash = await crypto.subtle.digest("SHA-1", new TextEncoder().encode(key + GUID));
  return btoa(String.fromCharCode(...new Uint8Array(hash)));
}
​
console.log("HTTP/1.1 101 Switching Protocols");
console.log("Upgrade: websocket");
console.log("Sec-WebSocket-Accept: " + await acceptFor("dGhlIHNhbXBsZSBub25jZQ=="));

// A pretend WebSocket: the two ends of one open connection, in this tab.
function socketPair() {
  const a = new EventTarget(), b = new EventTarget();
  const deliver = (to) => (data) => setTimeout(() => {
    const e = new Event("message");
    e.data = data;
    to.dispatchEvent(e);
  }, 5);
  a.send = deliver(b);
  b.send = deliver(a);
  return [a, b];
}
​
const [client, server] = socketPair();
server.addEventListener("message", (e) => console.log("server got:", e.data));
client.addEventListener("message", (e) => console.log("client got:", e.data));
​
client.send("subscribe: prices");
let price = 100;
const ticker = setInterval(() => {        // the server speaks first, unasked
  price += 1;
  server.send(JSON.stringify({ price }));
}, 100);
setTimeout(() => { clearInterval(ticker); console.log("closed"); }, 450);
console.log("connection open; nothing has arrived yet");

// GraphQL's core idea in 25 lines: the query is a shape, and the server
// walks it, calling a resolver only for the fields that were asked for.
let authorLookups = 0;
const data = {
  book: {
    title: "Dune", year: 1965, pages: 412,
    author: () => { authorLookups++; return { name: "Frank Herbert", born: 1920 }; },
  },
};
​
function parse(tokens) {                 // "{ a b { c } }" → { a: true, b: { c: true } }
  const shape = {};
  tokens.shift();                        // "{"
  while (tokens[0] !== "}") {
    const name = tokens.shift();
    shape[name] = tokens[0] === "{" ? parse(tokens) : true;
  }
  tokens.shift();                        // "}"
  return shape;
}
function run(value, shape) {
  const out = {};
  for (const [field, sub] of Object.entries(shape)) {
    const v = typeof value[field] === "function" ? value[field]() : value[field];
    out[field] = sub === true ? v : run(v, sub);
  }
  return out;
}
​
const ask = (q) => JSON.stringify(run(data, parse(q.match(/[{}]|\w+/g))));
console.log(ask("{ book { title author { name } } }"));
console.log(ask("{ book { title year } }"));
console.log("author resolver ran", authorLookups, "time(s)");

// gRPC sends Protocol Buffers: compact binary, not text.
// Field 1 holding 150 encodes to just three bytes.
function varint(n) {
  const bytes = [];
  while (n >= 0x80) {
    bytes.push((n & 0x7f) | 0x80);     // 7 bits, plus "more follows"
    n >>>= 7;
  }
  bytes.push(n);
  return bytes;
}
function field(number, value) {        // wire type 0: varint
  return [...varint((number << 3) | 0), ...varint(value)];
}
​
const hex = (bs) => bs.map((b) => b.toString(16).padStart(2, "0")).join(" ");
const json = JSON.stringify({ id: 150 });
console.log("protobuf:", hex(field(1, 150)), "=", field(1, 150).length, "bytes");
console.log("JSON:    ", json, "=", json.length, "bytes");