opentracker

opentracker/crates/otserver/src/limits.rs
limits.rs
⎇
Raw
1//! Abuse handling for the open UDP port.
2//!
3//! Layered in packet-touch order, cheapest check first, so that spending CPU on a
4//! packet is always justified by the packet having survived everything cheaper:
5//!
6//! 1. length / version / type filter (in `otproto`, no state at all)
7//! 2. per-IP token bucket
8//! 3. unknown-token blocklist
9//! 4. per-token token bucket
10//! 5. AEAD verification — the first genuinely expensive step (~1 µs)
11//! 6. AEAD-failure blocklist
12//! 7. writer saturation → `THROTTLE`
13//!
14//! Two cross-cutting rules:
15//!
16//! * **Never reply to an unauthenticated packet.** Not with an error, not with a
17//! `NACK`. Anything else makes the port a reflector and a forgery oracle.
18//! * **The rate limiter must not itself be a memory-exhaustion vector.** Every map
19//! here is capacity-capped and swept, because an attacker chooses the keys.
20
21use std::net::IpAddr;
22use std::num::NonZeroU32;
23use std::time::{Duration, Instant};
24
25use dashmap::DashMap;
26use governor::clock::DefaultClock;
27use governor::state::keyed::DefaultKeyedStateStore;
28use governor::state::{InMemoryState, NotKeyed};
29use governor::{Quota, RateLimiter};
30
31/// Sustained packets per second from one IP, and how many may arrive at once.
32/// Generous: a single IP can be a whole household behind NAT, all flushing queues
33/// after a dead spot.
34const IP_RATE: u32 = 25;
35const IP_BURST: u32 = 75;
36
37/// Per token. One report per second is already far above any profile; the burst
38/// covers a queue flush after an offline stretch.
39const TOKEN_RATE: u32 = 3;
40const TOKEN_BURST: u32 = 40;
41
42/// Revocation notices, the one reply sent without authenticating the request.
43///
44/// These numbers are the entire mitigation, so they are deliberately mean. The
45/// key is the *destination* address, which for a spoofed datagram is the victim:
46/// one packet a minute is a message, not a flood. A revoked device needs exactly
47/// one to act, and it retries on its own schedule anyway.
48///
49/// The global ceiling bounds a distributed spoof, where each victim stays under
50/// the per-address limit but the server is still made to emit broadly.
51const NOTICE_PER_ADDR_PER_MINUTE: u32 = 1;
52const NOTICE_GLOBAL_RATE: u32 = 10;
53const NOTICE_GLOBAL_BURST: u32 = 20;
54
55/// Strikes within [`STRIKE_WINDOW`] before an IP is banned for [`BAN_DURATION`].
56const UNKNOWN_TOKEN_STRIKES: u32 = 50;
57const AEAD_FAIL_STRIKES: u32 = 50;
58const STRIKE_WINDOW: Duration = Duration::from_secs(60);
59const BAN_DURATION: Duration = Duration::from_secs(600);
60
61/// Hard ceiling on tracked IPs. Reached only under attack; when it is, the map is
62/// swept rather than grown, because unbounded growth is the vulnerability.
63const MAX_TRACKED_IPS: usize = 100_000;
64
65type Keyed<K> = RateLimiter<K, DefaultKeyedStateStore<K>, DefaultClock>;
66type Direct = RateLimiter<NotKeyed, InMemoryState, DefaultClock>;
67
68fn quota(rate: u32, burst: u32) -> Quota {
69 Quota::per_second(NonZeroU32::new(rate).expect("rate is a non-zero literal"))
70 .allow_burst(NonZeroU32::new(burst).expect("burst is a non-zero literal"))
71}
72
73#[derive(Debug, Default)]
74struct Strikes {
75 count: u32,
76 window_started: Option<Instant>,
77 banned_until: Option<Instant>,
78}
79
80pub struct Limits {
81 per_ip: Keyed<IpAddr>,
82 per_token: Keyed<u64>,
83 notice_per_addr: Keyed<IpAddr>,
84 notice_global: Direct,
85 unknown_token: DashMap<IpAddr, Strikes>,
86 aead_fail: DashMap<IpAddr, Strikes>,
87}
88
89/// Why a packet was dropped. Used for aggregate counters only — never logged per
90/// packet, because per-packet logging is itself a denial-of-service amplifier.
91#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
92pub enum Drop {
93 IpRate,
94 TokenRate,
95 Banned,
96 /// The unauthenticated-notice budget, per destination or global.
97 NoticeRate,
98}
99
100impl Default for Limits {
101 fn default() -> Self {
102 Self::new()
103 }
104}
105
106impl Limits {
107 pub fn new() -> Self {
108 Self {
109 per_ip: RateLimiter::keyed(quota(IP_RATE, IP_BURST)),
110 per_token: RateLimiter::keyed(quota(TOKEN_RATE, TOKEN_BURST)),
111 notice_per_addr: RateLimiter::keyed(Quota::per_minute(
112 NonZeroU32::new(NOTICE_PER_ADDR_PER_MINUTE).expect("non-zero literal"),
113 )),
114 notice_global: RateLimiter::direct(quota(NOTICE_GLOBAL_RATE, NOTICE_GLOBAL_BURST)),
115 unknown_token: DashMap::new(),
116 aead_fail: DashMap::new(),
117 }
118 }
119
120 /// Step 2 and 3: is this source allowed to send anything right now?
121 pub fn check_ip(&self, ip: IpAddr) -> Result<(), Drop> {
122 if self.is_banned(ip) {
123 return Err(Drop::Banned);
124 }
125 self.per_ip.check_key(&ip).map_err(|_| Drop::IpRate)
126 }
127
128 /// Per-credential budget, applied *after* the AEAD check.
129 ///
130 /// Deliberately not before it, even though that would be cheaper. Exceeding
131 /// this budget produces a NACK, and a reply to an unverified datagram is a
132 /// reflector: `token_id` is cleartext, so an attacker can name a real token,
133 /// spoof a victim's source address, and have the server packet the victim.
134 /// The per-IP budget is the cheap pre-crypto filter; this one is not.
135 pub fn check_token(&self, token_id: u64) -> Result<(), Drop> {
136 self.per_token
137 .check_key(&token_id)
138 .map_err(|_| Drop::TokenRate)
139 }
140
141 /// Called when a datagram names a token that does not exist.
142 ///
143 /// Scanning for valid token ids is the cheapest attack against this design —
144 /// no key needed, just 2⁶⁴ guesses — so it is met with a ban rather than a
145 /// reply. The port stays silent either way.
146 pub fn note_unknown_token(&self, ip: IpAddr) {
147 self.strike(&self.unknown_token, ip, UNKNOWN_TOKEN_STRIKES);
148 }
149
150 /// Called when a datagram named a real token but failed AEAD. Either
151 /// corruption or forgery; both mean stop listening to this source.
152 pub fn note_aead_failure(&self, ip: IpAddr) {
153 self.strike(&self.aead_fail, ip, AEAD_FAIL_STRIKES);
154 }
155
156 fn is_banned(&self, ip: IpAddr) -> bool {
157 let now = Instant::now();
158 let banned = |m: &DashMap<IpAddr, Strikes>| {
159 m.get(&ip)
160 .and_then(|s| s.banned_until)
161 .is_some_and(|t| t > now)
162 };
163 banned(&self.unknown_token) || banned(&self.aead_fail)
164 }
165
166 fn strike(&self, map: &DashMap<IpAddr, Strikes>, ip: IpAddr, threshold: u32) {
167 // Sweep before inserting a new key, so the map can never exceed the cap.
168 if map.len() >= MAX_TRACKED_IPS && !map.contains_key(&ip) {
169 sweep(map);
170 if map.len() >= MAX_TRACKED_IPS {
171 // Still full after sweeping: we are under a wide attack. Dropping
172 // the strike is the right failure mode — the per-IP rate limiter
173 // is still holding, and growing this map is what the attacker
174 // wants.
175 return;
176 }
177 }
178
179 let now = Instant::now();
180 let mut entry = map.entry(ip).or_default();
181 match entry.window_started {
182 Some(started) if now.duration_since(started) <= STRIKE_WINDOW => entry.count += 1,
183 _ => {
184 entry.window_started = Some(now);
185 entry.count = 1;
186 }
187 }
188 if entry.count >= threshold {
189 entry.banned_until = Some(now + BAN_DURATION);
190 entry.count = 0;
191 entry.window_started = None;
192 }
193 }
194
195 /// May we send an unauthenticated revocation notice to `addr` right now?
196 ///
197 /// Both budgets are checked, and both must pass. This is called with the
198 /// address the datagram *claimed* to come from, which is exactly the point:
199 /// an attacker spoofing a victim gets the victim's budget, not their own.
200 pub fn check_notice(&self, addr: IpAddr) -> Result<(), Drop> {
201 if self.notice_global.check().is_err() {
202 return Err(Drop::NoticeRate);
203 }
204 self.notice_per_addr
205 .check_key(&addr)
206 .map_err(|_| Drop::NoticeRate)
207 }
208
209 /// Periodic maintenance. Drops expired bans and stale rate-limiter state so
210 /// idle keys do not accumulate.
211 pub fn gc(&self) {
212 self.per_ip.retain_recent();
213 self.per_token.retain_recent();
214 self.notice_per_addr.retain_recent();
215 sweep(&self.unknown_token);
216 sweep(&self.aead_fail);
217 }
218
219 /// For `/metrics`.
220 pub fn tracked_ips(&self) -> usize {
221 self.unknown_token.len() + self.aead_fail.len()
222 }
223}
224
225fn sweep(map: &DashMap<IpAddr, Strikes>) {
226 let now = Instant::now();
227 map.retain(|_, s| {
228 let ban_active = s.banned_until.is_some_and(|t| t > now);
229 let window_active = s
230 .window_started
231 .is_some_and(|t| now.duration_since(t) <= STRIKE_WINDOW);
232 ban_active || window_active
233 });
234}
235
236#[cfg(test)]
237mod tests {
238 use super::*;
239
240 const IP: IpAddr = IpAddr::V4(std::net::Ipv4Addr::new(203, 0, 113, 5));
241
242 #[test]
243 fn a_burst_is_allowed_then_the_rate_bites() {
244 let limits = Limits::new();
245 for i in 0..IP_BURST {
246 assert!(
247 limits.check_ip(IP).is_ok(),
248 "packet {i} of the burst was dropped"
249 );
250 }
251 assert_eq!(limits.check_ip(IP), Err(Drop::IpRate));
252 }
253
254 #[test]
255 fn per_token_limits_are_independent_of_each_other() {
256 let limits = Limits::new();
257 for _ in 0..TOKEN_BURST {
258 assert!(limits.check_token(1).is_ok());
259 }
260 assert_eq!(limits.check_token(1), Err(Drop::TokenRate));
261 // A different token is unaffected: one noisy phone must not silence a
262 // whole household behind the same NAT.
263 assert!(limits.check_token(2).is_ok());
264 }
265
266 #[test]
267 fn token_scanning_earns_a_ban() {
268 let limits = Limits::new();
269 for _ in 0..UNKNOWN_TOKEN_STRIKES {
270 limits.note_unknown_token(IP);
271 }
272 assert_eq!(limits.check_ip(IP), Err(Drop::Banned));
273 }
274
275 #[test]
276 fn forged_packets_earn_a_ban() {
277 let limits = Limits::new();
278 for _ in 0..AEAD_FAIL_STRIKES {
279 limits.note_aead_failure(IP);
280 }
281 assert_eq!(limits.check_ip(IP), Err(Drop::Banned));
282 }
283
284 #[test]
285 fn a_ban_is_specific_to_the_offending_address() {
286 let limits = Limits::new();
287 for _ in 0..UNKNOWN_TOKEN_STRIKES {
288 limits.note_unknown_token(IP);
289 }
290 let other = IpAddr::V4(std::net::Ipv4Addr::new(198, 51, 100, 9));
291 assert!(limits.check_ip(other).is_ok());
292 }
293
294 #[test]
295 fn strikes_below_the_threshold_do_not_ban() {
296 let limits = Limits::new();
297 for _ in 0..UNKNOWN_TOKEN_STRIKES - 1 {
298 limits.note_unknown_token(IP);
299 }
300 assert!(limits.check_ip(IP).is_ok());
301 }
302
303 #[test]
304 fn gc_drops_entries_with_nothing_left_to_remember() {
305 let limits = Limits::new();
306 limits.note_unknown_token(IP);
307 assert_eq!(limits.tracked_ips(), 1);
308
309 // Force the window to look expired rather than sleeping for a minute.
310 limits
311 .unknown_token
312 .get_mut(&IP)
313 .expect("entry")
314 .window_started = Some(Instant::now() - STRIKE_WINDOW * 2);
315 limits.gc();
316 assert_eq!(limits.tracked_ips(), 0);
317 }
318
319 #[test]
320 fn a_ban_survives_gc_until_it_expires() {
321 let limits = Limits::new();
322 for _ in 0..UNKNOWN_TOKEN_STRIKES {
323 limits.note_unknown_token(IP);
324 }
325 limits.gc();
326 assert_eq!(
327 limits.check_ip(IP),
328 Err(Drop::Banned),
329 "gc must not lift an active ban"
330 );
331 }
332}
333