Crypto recipe

Streaming & large-file encryption

Encrypt data too big to hold in memory — chunk it, authenticate every chunk, and bind the order.

AES-GCM and the other one-shot AEADs assume the whole message fits in memory under a single nonce — and GCM has a hard limit of roughly 64 GiB per key+nonce. For a large file or a live stream you encrypt in fixed-size chunks instead, authenticating each chunk on its own.

Chunking safely is the hard part. Each chunk needs a unique nonce (a counter works under a per-file key), and the scheme has to stop an attacker from reordering, dropping, duplicating, or truncating chunks — so bind each chunk's position and mark the final one. libsodium's secretstream and Rust's aead::stream do all of this for you; with plain AES-GCM you do it by hand, as shown here.

Get it right

  • Use a fresh key per file (or a per-file random nonce prefix) so chunk counters never collide across files.
  • Give every chunk a unique nonce — a monotonic counter under that per-file key — and never reuse one.
  • Authenticate order and the end of the stream — a counter in the nonce plus a 'final' flag in the AAD — so chunks can't be reordered, dropped, or the stream truncated.
  • Prefer a purpose-built construction — libsodium secretstream, the STREAM construction (Rust aead::stream), age, or Tink streaming AEAD — over hand-rolling when you can.
  • Pick a chunk size (e.g. 16–64 KiB) that balances memory use against per-chunk tag overhead.

Implementation

Setup Standard library (crypto/aes, crypto/cipher).

Go
package main

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"encoding/binary"
	"fmt"
)

func main() {
	key := make([]byte, 32)
	rand.Read(key) // one fresh key per file (derive or wrap it in practice)
	block, _ := aes.NewCipher(key)
	gcm, _ := cipher.NewGCM(block)

	chunks := [][]byte{[]byte("chunk one "), []byte("chunk two "), []byte("the last chunk")}

	// Encrypt: nonce = chunk counter; a 'final' flag in the AAD means the stream
	// cannot be silently truncated or extended. This hand-rolls what
	// secretstream / STREAM automate.
	aad := func(final bool) []byte { if final { return []byte{1} }; return []byte{0} }
	nonce := func(i int) []byte {
		n := make([]byte, gcm.NonceSize())
		binary.BigEndian.PutUint64(n[gcm.NonceSize()-8:], uint64(i))
		return n
	}
	var enc [][]byte
	for i, c := range chunks {
		enc = append(enc, gcm.Seal(nil, nonce(i), c, aad(i == len(chunks)-1)))
	}

	// Decrypt: same nonce sequence + final flag; any tamper or reorder fails.
	var plain []byte
	for i, ct := range enc {
		m, err := gcm.Open(nil, nonce(i), ct, aad(i == len(enc)-1))
		if err != nil {
			panic("forged or reordered chunk")
		}
		plain = append(plain, m...)
	}
	fmt.Println(string(plain))
}

The counter nonce + final-chunk AAD flag is the manual version of what secretstream / STREAM do for you.

Setup pip install cryptography.

Python
from cryptography.hazmat.primitives.ciphers.aead import AESGCM

key = AESGCM.generate_key(bit_length=256)  # one fresh key per file
aead = AESGCM(key)

chunks = [b"chunk one ", b"chunk two ", b"the last chunk"]

def nonce(i):
    return i.to_bytes(12, "big")           # per-chunk counter nonce

# Encrypt: AAD flags the final chunk so the stream can't be truncated/extended.
encrypted = [aead.encrypt(nonce(i), c, b"\x01" if i == len(chunks) - 1 else b"\x00")
             for i, c in enumerate(chunks)]

# Decrypt: same nonce + flag; a bad tag or wrong order raises InvalidTag.
plain = b"".join(aead.decrypt(nonce(i), ct, b"\x01" if i == len(encrypted) - 1 else b"\x00")
                 for i, ct in enumerate(encrypted))

assert plain == b"chunk one chunk two the last chunk"

Setup Built in (node:crypto).

Node.js
import { randomBytes, createCipheriv, createDecipheriv } from 'node:crypto';

const key = randomBytes(32); // one fresh key per file
const chunks = ['chunk one ', 'chunk two ', 'the last chunk'].map((s) => Buffer.from(s));

const nonce = (i) => { const n = Buffer.alloc(12); n.writeUInt32BE(i, 8); return n; };
const aad = (i, n) => Buffer.from([i === n - 1 ? 1 : 0]); // final-chunk flag

// Encrypt each chunk under its counter nonce; keep the per-chunk tag.
const enc = chunks.map((chunk, i) => {
  const c = createCipheriv('aes-256-gcm', key, nonce(i));
  c.setAAD(aad(i, chunks.length));
  const body = Buffer.concat([c.update(chunk), c.final()]);
  return { body, tag: c.getAuthTag() };
});

// Decrypt: same nonce + flag; final() throws if a chunk is forged or reordered.
let plain = Buffer.alloc(0);
enc.forEach(({ body, tag }, i) => {
  const d = createDecipheriv('aes-256-gcm', key, nonce(i));
  d.setAAD(aad(i, enc.length));
  d.setAuthTag(tag);
  plain = Buffer.concat([plain, d.update(body), d.final()]);
});

if (plain.toString() !== 'chunk one chunk two the last chunk') throw new Error('mismatch');
console.log(plain.toString());

Setup Built in (System.Security.Cryptography.AesGcm).

.NET
using System.Buffers.Binary;
using System.Security.Cryptography;
using System.Text;

byte[] key = RandomNumberGenerator.GetBytes(32); // one fresh key per file
string[] parts = { "chunk one ", "chunk two ", "the last chunk" };

static byte[] Nonce(int i) {
    byte[] n = new byte[12];
    BinaryPrimitives.WriteUInt32BigEndian(n.AsSpan(8), (uint)i); // per-chunk counter
    return n;
}

// Encrypt: per-chunk counter nonce; AAD byte flags the final chunk.
var enc = new List<(byte[] ct, byte[] tag)>();
using (var aes = new AesGcm(key, 16)) {
    for (int i = 0; i < parts.Length; i++) {
        byte[] pt = Encoding.UTF8.GetBytes(parts[i]);
        byte[] ct = new byte[pt.Length], tag = new byte[16];
        aes.Encrypt(Nonce(i), pt, ct, tag, new[] { (byte)(i == parts.Length - 1 ? 1 : 0) });
        enc.Add((ct, tag));
    }
}

// Decrypt: same nonce + flag; Decrypt throws if a chunk is forged or reordered.
var plain = new List<byte>();
using (var aes = new AesGcm(key, 16)) {
    for (int i = 0; i < enc.Count; i++) {
        byte[] pt = new byte[enc[i].ct.Length];
        aes.Decrypt(Nonce(i), enc[i].ct, enc[i].tag, pt, new[] { (byte)(i == enc.Count - 1 ? 1 : 0) });
        plain.AddRange(pt);
    }
}

if (Encoding.UTF8.GetString(plain.ToArray()) != "chunk one chunk two the last chunk")
    throw new Exception("mismatch");
Console.WriteLine(Encoding.UTF8.GetString(plain.ToArray()));

Setup libsodium (crypto_secretstream_xchacha20poly1305).

C++
#include <sodium.h>
#include <string>
#include <vector>

int main() {
    if (sodium_init() < 0) return 1;

    unsigned char key[crypto_secretstream_xchacha20poly1305_KEYBYTES];
    crypto_secretstream_xchacha20poly1305_keygen(key);

    std::vector<std::string> chunks = {"chunk one ", "chunk two ", "the last chunk"};

    // Encrypt: push each chunk; TAG_FINAL marks the end so truncation is detected.
    crypto_secretstream_xchacha20poly1305_state st;
    unsigned char header[crypto_secretstream_xchacha20poly1305_HEADERBYTES];
    crypto_secretstream_xchacha20poly1305_init_push(&st, header, key); // store header with the file

    std::vector<std::vector<unsigned char>> out;
    for (size_t i = 0; i < chunks.size(); i++) {
        unsigned char tag = (i + 1 == chunks.size())
            ? crypto_secretstream_xchacha20poly1305_TAG_FINAL : 0;
        std::vector<unsigned char> c(chunks[i].size() + crypto_secretstream_xchacha20poly1305_ABYTES);
        unsigned long long clen;
        crypto_secretstream_xchacha20poly1305_push(&st, c.data(), &clen,
            reinterpret_cast<const unsigned char *>(chunks[i].data()), chunks[i].size(),
            nullptr, 0, tag);
        c.resize(clen);
        out.push_back(c);
    }

    // Decrypt: pull each chunk; the stream is complete only when TAG_FINAL appears.
    crypto_secretstream_xchacha20poly1305_state dst;
    if (crypto_secretstream_xchacha20poly1305_init_pull(&dst, header, key) != 0) return 1;
    std::string plain;
    bool final_seen = false;
    for (auto &c : out) {
        std::vector<unsigned char> m(c.size());
        unsigned long long mlen;
        unsigned char tag;
        if (crypto_secretstream_xchacha20poly1305_pull(&dst, m.data(), &mlen, &tag,
                c.data(), c.size(), nullptr, 0) != 0) {
            return 1; // forged or reordered
        }
        plain.append(reinterpret_cast<char *>(m.data()), mlen);
        if (tag == crypto_secretstream_xchacha20poly1305_TAG_FINAL) final_seen = true;
    }
    return (final_seen && plain == "chunk one chunk two the last chunk") ? 0 : 1;
}

secretstream handles per-chunk nonces and key rotation for you; TAG_FINAL seals the stream so a truncated file is rejected.

Setup Built in (javax.crypto).

Java
import javax.crypto.Cipher;
import javax.crypto.spec.GCMParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import java.io.ByteArrayOutputStream;
import java.nio.ByteBuffer;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.List;

byte[] keyBytes = new byte[32];
new SecureRandom().nextBytes(keyBytes); // one fresh key per file
SecretKeySpec key = new SecretKeySpec(keyBytes, "AES");

String[] parts = { "chunk one ", "chunk two ", "the last chunk" };

// Encrypt: nonce = 12-byte big-endian chunk counter; AAD flags the final chunk.
List<byte[]> enc = new ArrayList<>();
for (int i = 0; i < parts.length; i++) {
    byte[] nonce = ByteBuffer.allocate(12).putInt(8, i).array();
    Cipher c = Cipher.getInstance("AES/GCM/NoPadding");
    c.init(Cipher.ENCRYPT_MODE, key, new GCMParameterSpec(128, nonce));
    c.updateAAD(new byte[] { (byte) (i == parts.length - 1 ? 1 : 0) });
    enc.add(c.doFinal(parts[i].getBytes()));
}

// Decrypt: same nonce + flag; doFinal throws AEADBadTagException on any tamper.
ByteArrayOutputStream plain = new ByteArrayOutputStream();
for (int i = 0; i < enc.size(); i++) {
    byte[] nonce = ByteBuffer.allocate(12).putInt(8, i).array();
    Cipher c = Cipher.getInstance("AES/GCM/NoPadding");
    c.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(128, nonce));
    c.updateAAD(new byte[] { (byte) (i == enc.size() - 1 ? 1 : 0) });
    plain.writeBytes(c.doFinal(enc.get(i)));
}

if (!plain.toString().equals("chunk one chunk two the last chunk"))
    throw new RuntimeException("mismatch");
System.out.println(plain.toString());

Setup Cargo.toml: aes-gcm = { version = "0.10", features = ["stream"] }.

Rust
use aes_gcm::aead::generic_array::GenericArray;
use aes_gcm::aead::stream::{DecryptorBE32, EncryptorBE32};
use aes_gcm::aead::{rand_core::RngCore, OsRng};
use aes_gcm::Aes256Gcm;

fn main() {
    let mut key = [0u8; 32];
    let mut nonce = [0u8; 7]; // STREAM nonce; a 5-byte counter is appended internally
    OsRng.fill_bytes(&mut key);
    OsRng.fill_bytes(&mut nonce);

    let parts: [&[u8]; 3] = [b"chunk one ", b"chunk two ", b"the last chunk"];

    // Encrypt: encrypt_next per chunk, encrypt_last to seal the end of the stream.
    let mut enc = EncryptorBE32::<Aes256Gcm>::new(
        GenericArray::from_slice(&key), GenericArray::from_slice(&nonce));
    let c0 = enc.encrypt_next(parts[0]).unwrap();
    let c1 = enc.encrypt_next(parts[1]).unwrap();
    let c2 = enc.encrypt_last(parts[2]).unwrap();

    // Decrypt: decrypt_next then decrypt_last, in order; any tamper returns Err.
    let mut dec = DecryptorBE32::<Aes256Gcm>::new(
        GenericArray::from_slice(&key), GenericArray::from_slice(&nonce));
    let mut plain = Vec::new();
    plain.extend(dec.decrypt_next(c0.as_slice()).unwrap());
    plain.extend(dec.decrypt_next(c1.as_slice()).unwrap());
    plain.extend(dec.decrypt_last(c2.as_slice()).unwrap());

    assert_eq!(plain, b"chunk one chunk two the last chunk");
}

The STREAM construction (EncryptorBE32) manages the per-chunk counter; encrypt_last marks the end of the stream.