Crypto recipe
Streaming & large-file encryption
Encrypt data too big to hold in memory — chunk it, authenticate every chunk, and bind the order.
AES-GCM and the other one-shot AEADs assume the whole message fits in memory under a single nonce — and GCM has a hard limit of roughly 64 GiB per key+nonce. For a large file or a live stream you encrypt in fixed-size chunks instead, authenticating each chunk on its own.
Chunking safely is the hard part. Each chunk needs a unique nonce (a counter works under a per-file key), and the scheme has to stop an attacker from reordering, dropping, duplicating, or truncating chunks — so bind each chunk's position and mark the final one. libsodium's secretstream and Rust's aead::stream do all of this for you; with plain AES-GCM you do it by hand, as shown here.
Get it right
- Use a fresh key per file (or a per-file random nonce prefix) so chunk counters never collide across files.
- Give every chunk a unique nonce — a monotonic counter under that per-file key — and never reuse one.
- Authenticate order and the end of the stream — a counter in the nonce plus a 'final' flag in the AAD — so chunks can't be reordered, dropped, or the stream truncated.
- Prefer a purpose-built construction — libsodium
secretstream, the STREAM construction (Rustaead::stream),age, or Tink streaming AEAD — over hand-rolling when you can. - Pick a chunk size (e.g. 16–64 KiB) that balances memory use against per-chunk tag overhead.
Implementation
Setup Standard library (crypto/aes, crypto/cipher).
package main
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/binary"
"fmt"
)
func main() {
key := make([]byte, 32)
rand.Read(key) // one fresh key per file (derive or wrap it in practice)
block, _ := aes.NewCipher(key)
gcm, _ := cipher.NewGCM(block)
chunks := [][]byte{[]byte("chunk one "), []byte("chunk two "), []byte("the last chunk")}
// Encrypt: nonce = chunk counter; a 'final' flag in the AAD means the stream
// cannot be silently truncated or extended. This hand-rolls what
// secretstream / STREAM automate.
aad := func(final bool) []byte { if final { return []byte{1} }; return []byte{0} }
nonce := func(i int) []byte {
n := make([]byte, gcm.NonceSize())
binary.BigEndian.PutUint64(n[gcm.NonceSize()-8:], uint64(i))
return n
}
var enc [][]byte
for i, c := range chunks {
enc = append(enc, gcm.Seal(nil, nonce(i), c, aad(i == len(chunks)-1)))
}
// Decrypt: same nonce sequence + final flag; any tamper or reorder fails.
var plain []byte
for i, ct := range enc {
m, err := gcm.Open(nil, nonce(i), ct, aad(i == len(enc)-1))
if err != nil {
panic("forged or reordered chunk")
}
plain = append(plain, m...)
}
fmt.Println(string(plain))
} The counter nonce + final-chunk AAD flag is the manual version of what secretstream / STREAM do for you.
Setup pip install cryptography.
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
key = AESGCM.generate_key(bit_length=256) # one fresh key per file
aead = AESGCM(key)
chunks = [b"chunk one ", b"chunk two ", b"the last chunk"]
def nonce(i):
return i.to_bytes(12, "big") # per-chunk counter nonce
# Encrypt: AAD flags the final chunk so the stream can't be truncated/extended.
encrypted = [aead.encrypt(nonce(i), c, b"\x01" if i == len(chunks) - 1 else b"\x00")
for i, c in enumerate(chunks)]
# Decrypt: same nonce + flag; a bad tag or wrong order raises InvalidTag.
plain = b"".join(aead.decrypt(nonce(i), ct, b"\x01" if i == len(encrypted) - 1 else b"\x00")
for i, ct in enumerate(encrypted))
assert plain == b"chunk one chunk two the last chunk" Setup Built in (node:crypto).
import { randomBytes, createCipheriv, createDecipheriv } from 'node:crypto';
const key = randomBytes(32); // one fresh key per file
const chunks = ['chunk one ', 'chunk two ', 'the last chunk'].map((s) => Buffer.from(s));
const nonce = (i) => { const n = Buffer.alloc(12); n.writeUInt32BE(i, 8); return n; };
const aad = (i, n) => Buffer.from([i === n - 1 ? 1 : 0]); // final-chunk flag
// Encrypt each chunk under its counter nonce; keep the per-chunk tag.
const enc = chunks.map((chunk, i) => {
const c = createCipheriv('aes-256-gcm', key, nonce(i));
c.setAAD(aad(i, chunks.length));
const body = Buffer.concat([c.update(chunk), c.final()]);
return { body, tag: c.getAuthTag() };
});
// Decrypt: same nonce + flag; final() throws if a chunk is forged or reordered.
let plain = Buffer.alloc(0);
enc.forEach(({ body, tag }, i) => {
const d = createDecipheriv('aes-256-gcm', key, nonce(i));
d.setAAD(aad(i, enc.length));
d.setAuthTag(tag);
plain = Buffer.concat([plain, d.update(body), d.final()]);
});
if (plain.toString() !== 'chunk one chunk two the last chunk') throw new Error('mismatch');
console.log(plain.toString()); Setup Built in (System.Security.Cryptography.AesGcm).
using System.Buffers.Binary;
using System.Security.Cryptography;
using System.Text;
byte[] key = RandomNumberGenerator.GetBytes(32); // one fresh key per file
string[] parts = { "chunk one ", "chunk two ", "the last chunk" };
static byte[] Nonce(int i) {
byte[] n = new byte[12];
BinaryPrimitives.WriteUInt32BigEndian(n.AsSpan(8), (uint)i); // per-chunk counter
return n;
}
// Encrypt: per-chunk counter nonce; AAD byte flags the final chunk.
var enc = new List<(byte[] ct, byte[] tag)>();
using (var aes = new AesGcm(key, 16)) {
for (int i = 0; i < parts.Length; i++) {
byte[] pt = Encoding.UTF8.GetBytes(parts[i]);
byte[] ct = new byte[pt.Length], tag = new byte[16];
aes.Encrypt(Nonce(i), pt, ct, tag, new[] { (byte)(i == parts.Length - 1 ? 1 : 0) });
enc.Add((ct, tag));
}
}
// Decrypt: same nonce + flag; Decrypt throws if a chunk is forged or reordered.
var plain = new List<byte>();
using (var aes = new AesGcm(key, 16)) {
for (int i = 0; i < enc.Count; i++) {
byte[] pt = new byte[enc[i].ct.Length];
aes.Decrypt(Nonce(i), enc[i].ct, enc[i].tag, pt, new[] { (byte)(i == enc.Count - 1 ? 1 : 0) });
plain.AddRange(pt);
}
}
if (Encoding.UTF8.GetString(plain.ToArray()) != "chunk one chunk two the last chunk")
throw new Exception("mismatch");
Console.WriteLine(Encoding.UTF8.GetString(plain.ToArray())); Setup libsodium (crypto_secretstream_xchacha20poly1305).
#include <sodium.h>
#include <string>
#include <vector>
int main() {
if (sodium_init() < 0) return 1;
unsigned char key[crypto_secretstream_xchacha20poly1305_KEYBYTES];
crypto_secretstream_xchacha20poly1305_keygen(key);
std::vector<std::string> chunks = {"chunk one ", "chunk two ", "the last chunk"};
// Encrypt: push each chunk; TAG_FINAL marks the end so truncation is detected.
crypto_secretstream_xchacha20poly1305_state st;
unsigned char header[crypto_secretstream_xchacha20poly1305_HEADERBYTES];
crypto_secretstream_xchacha20poly1305_init_push(&st, header, key); // store header with the file
std::vector<std::vector<unsigned char>> out;
for (size_t i = 0; i < chunks.size(); i++) {
unsigned char tag = (i + 1 == chunks.size())
? crypto_secretstream_xchacha20poly1305_TAG_FINAL : 0;
std::vector<unsigned char> c(chunks[i].size() + crypto_secretstream_xchacha20poly1305_ABYTES);
unsigned long long clen;
crypto_secretstream_xchacha20poly1305_push(&st, c.data(), &clen,
reinterpret_cast<const unsigned char *>(chunks[i].data()), chunks[i].size(),
nullptr, 0, tag);
c.resize(clen);
out.push_back(c);
}
// Decrypt: pull each chunk; the stream is complete only when TAG_FINAL appears.
crypto_secretstream_xchacha20poly1305_state dst;
if (crypto_secretstream_xchacha20poly1305_init_pull(&dst, header, key) != 0) return 1;
std::string plain;
bool final_seen = false;
for (auto &c : out) {
std::vector<unsigned char> m(c.size());
unsigned long long mlen;
unsigned char tag;
if (crypto_secretstream_xchacha20poly1305_pull(&dst, m.data(), &mlen, &tag,
c.data(), c.size(), nullptr, 0) != 0) {
return 1; // forged or reordered
}
plain.append(reinterpret_cast<char *>(m.data()), mlen);
if (tag == crypto_secretstream_xchacha20poly1305_TAG_FINAL) final_seen = true;
}
return (final_seen && plain == "chunk one chunk two the last chunk") ? 0 : 1;
} secretstream handles per-chunk nonces and key rotation for you; TAG_FINAL seals the stream so a truncated file is rejected.
Setup Built in (javax.crypto).
import javax.crypto.Cipher;
import javax.crypto.spec.GCMParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import java.io.ByteArrayOutputStream;
import java.nio.ByteBuffer;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.List;
byte[] keyBytes = new byte[32];
new SecureRandom().nextBytes(keyBytes); // one fresh key per file
SecretKeySpec key = new SecretKeySpec(keyBytes, "AES");
String[] parts = { "chunk one ", "chunk two ", "the last chunk" };
// Encrypt: nonce = 12-byte big-endian chunk counter; AAD flags the final chunk.
List<byte[]> enc = new ArrayList<>();
for (int i = 0; i < parts.length; i++) {
byte[] nonce = ByteBuffer.allocate(12).putInt(8, i).array();
Cipher c = Cipher.getInstance("AES/GCM/NoPadding");
c.init(Cipher.ENCRYPT_MODE, key, new GCMParameterSpec(128, nonce));
c.updateAAD(new byte[] { (byte) (i == parts.length - 1 ? 1 : 0) });
enc.add(c.doFinal(parts[i].getBytes()));
}
// Decrypt: same nonce + flag; doFinal throws AEADBadTagException on any tamper.
ByteArrayOutputStream plain = new ByteArrayOutputStream();
for (int i = 0; i < enc.size(); i++) {
byte[] nonce = ByteBuffer.allocate(12).putInt(8, i).array();
Cipher c = Cipher.getInstance("AES/GCM/NoPadding");
c.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(128, nonce));
c.updateAAD(new byte[] { (byte) (i == enc.size() - 1 ? 1 : 0) });
plain.writeBytes(c.doFinal(enc.get(i)));
}
if (!plain.toString().equals("chunk one chunk two the last chunk"))
throw new RuntimeException("mismatch");
System.out.println(plain.toString()); Setup Cargo.toml: aes-gcm = { version = "0.10", features = ["stream"] }.
use aes_gcm::aead::generic_array::GenericArray;
use aes_gcm::aead::stream::{DecryptorBE32, EncryptorBE32};
use aes_gcm::aead::{rand_core::RngCore, OsRng};
use aes_gcm::Aes256Gcm;
fn main() {
let mut key = [0u8; 32];
let mut nonce = [0u8; 7]; // STREAM nonce; a 5-byte counter is appended internally
OsRng.fill_bytes(&mut key);
OsRng.fill_bytes(&mut nonce);
let parts: [&[u8]; 3] = [b"chunk one ", b"chunk two ", b"the last chunk"];
// Encrypt: encrypt_next per chunk, encrypt_last to seal the end of the stream.
let mut enc = EncryptorBE32::<Aes256Gcm>::new(
GenericArray::from_slice(&key), GenericArray::from_slice(&nonce));
let c0 = enc.encrypt_next(parts[0]).unwrap();
let c1 = enc.encrypt_next(parts[1]).unwrap();
let c2 = enc.encrypt_last(parts[2]).unwrap();
// Decrypt: decrypt_next then decrypt_last, in order; any tamper returns Err.
let mut dec = DecryptorBE32::<Aes256Gcm>::new(
GenericArray::from_slice(&key), GenericArray::from_slice(&nonce));
let mut plain = Vec::new();
plain.extend(dec.decrypt_next(c0.as_slice()).unwrap());
plain.extend(dec.decrypt_next(c1.as_slice()).unwrap());
plain.extend(dec.decrypt_last(c2.as_slice()).unwrap());
assert_eq!(plain, b"chunk one chunk two the last chunk");
} The STREAM construction (EncryptorBE32) manages the per-chunk counter; encrypt_last marks the end of the stream.