MythEclipse/omarchy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sync config from arch

Browse Source 
- hypr/apps.lua
- hypr/autostart.lua
- hypr/envs.lua
- hypr/hyprland.lua
- hypr/hyprsunset.conf
- hypr/input.lua
- hypr/looknfeel.lua
- hypr/omasettings.lua
- hypr/xdph.conf
- omarchy/branding/about.txt
- omarchy/branding/screensaver.txt
- omarchy/extensions/omarchy-menu.jsonc
- omarchy/hooks/battery-low.d/play-warning-sound.sample
- omarchy/hooks/font-set.d/show-font-notification.sample
- omarchy/hooks/post-boot.d/weather.sample
- omarchy/hooks/post-update.d/install-voxtype.hook
- omarchy/hooks/post-update.d/setup-agent.hook
- omarchy/hooks/post-update.d/setup-fingerprint.hook
- omarchy/hooks/post-update.d/show-update-notification.sample
- omarchy/hooks/pre-refresh-pacman.d/add-custom-repo.sample
- omarchy/hooks/theme-set.d/show-theme-notification.sample
- omarchy/shell.json
- omarchy/shell.toml
- omarchy/theme.name
- omarchy/themes/azure-glow/README.md
- omarchy/themes/azure-glow/alacritty.toml
- omarchy/themes/azure-glow/btop.theme
- omarchy/themes/azure-glow/hyprland.conf
- omarchy/themes/azure-glow/hyprlock.conf
- omarchy/themes/azure-glow/icons.theme
- … 269 more
This commit is contained in:
asepharyana
2026-09-23 15:19:12 +07:00
commit 1cdb82a76f
300 changed files with 78143 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/io.github.x3me.nexthop/nexthopd/__init__.py
+8
View File
@@ -0,0 +1,8 @@
"""nexthopd — the measurement daemon behind the Nexthop Omarchy plugin.
Standard library only, on purpose: the plugin is installed by cloning a git
repo, and the marketplace installer never builds or runs anything. Anything
that needed pip or a compiler would make installation a second, manual step.
"""
__version__ = "0.2.42"
plugins/io.github.x3me.nexthop/nexthopd/__main__.py
+5
View File
@@ -0,0 +1,5 @@
"""`python3 -m nexthopd` runs the daemon; the CLI lives at nexthopd.cli."""
import sys
from .daemon import main
sys.exit(main())
plugins/io.github.x3me.nexthop/nexthopd/apps.py
+403
View File
@@ -0,0 +1,403 @@
"""Per-application traffic, without root.
Linux only hands per-process byte counts to privileged tooling (pcap, eBPF)
— that is why nethogs needs root. The unprivileged truth available is
`ss -tinp`: every TCP socket's bytes_sent / bytes_received with the owning
process. Sampled on an interval, socket deltas aggregate into honest
per-app rates and running totals.
What this cannot see: UDP — and with it QUIC, which is how Chrome talks to
much of Google. That traffic surfaces as the gap between the interface
counters and the TCP sum, shown as its own "unattributed" bucket rather
than silently missing.
The same `ss` line also carries what the kernel already knows about each
connection's timing: `rtt:<srtt>/<rttvar>` (its smoothed round trip),
`minrtt` (the lowest it has ever seen on that path) and `bytes_retrans`.
That is a latency measurement of the user's own traffic, to the hosts they
actually talk to, costing no probe and no privilege — so we read it rather
than throw it away.
`minrtt` is the path's structural floor: distance, switching, serialisation.
`srtt - minrtt` is therefore queueing delay with distance divided out, and
every connection is its own control — which is what makes a 300 ms socket to
another continent comparable with a 5 ms socket next door. A raw `srtt` is
never a verdict on its own; only the difference travels.
"""
import os
import re
import select
import shutil
import statistics
import subprocess
import time
from collections import deque
from typing import NamedTuple
from .probes import nearest_rank
# users:(("chrome",pid=4958,fd=66)) — first process owning the socket.
RE_USER = re.compile(r'users:\(\("([^"]+)",pid=(\d+)')
RE_SENT = re.compile(r"bytes_sent:(\d+)")
RE_RECV = re.compile(r"bytes_received:(\d+)")
# rtt:<srtt>/<rttvar> — both milliseconds. minrtt is printed separately.
RE_RTT = re.compile(r"\brtt:([\d.]+)/([\d.]+)")
RE_MINRTT = re.compile(r"\bminrtt:([\d.]+)")
RE_RETRANS = re.compile(r"\bbytes_retrans:(\d+)")
class Sock(NamedTuple):
"""One socket's sample. Indexable, so older positional callers still work."""
app: str
pid: int
sent: int
recv: int
srtt: float = None # kernel smoothed RTT, ms
minrtt: float = None # lowest RTT seen on this path, ms
retrans: int = 0 # bytes retransmitted by the kernel
# A connection has to have carried something before its floor is worth
# trusting: `minrtt` is a minimum over the connection's life, so a socket
# that has only ever been busy may never have seen a quiet moment. Note the
# error direction — a floor biased high UNDER-states queueing, so this
# threshold is about honesty, not safety. Argue with the number, not the rule.
MIN_LATENCY_BYTES = 4096
# A plausibility ceiling, distinct from a sample-count floor: past this, the
# number describes a broken measurement rather than a slow link, and a
# measurement we cannot believe is worth less than no measurement.
MAX_PLAUSIBLE_RTT_MS = 10_000.0
# `ss` prints three decimals, so allow rounding before calling an inverted
# pair (floor above the average, which cannot happen) a stale field.
RTT_INVERSION_TOLERANCE_MS = 0.05
# Below this many qualifying sockets we publish nothing rather than a
# distribution drawn from a handful of connections.
MIN_LATENCY_SOCKETS = 3
def socket_timing(s) -> tuple:
"""(srtt, floor, queue) for one socket, or None if it does not qualify.
One guard, used by both the aggregate and the per-app medians, so a
socket rejected in one place cannot be silently counted in the other.
"""
if s.srtt is None or s.minrtt is None:
return None # kernel has no timing for it yet
if s.sent + s.recv < MIN_LATENCY_BYTES:
return None # too little traffic to trust the floor
if s.srtt <= 0 or s.minrtt <= 0:
return None
if s.srtt > MAX_PLAUSIBLE_RTT_MS or s.minrtt > MAX_PLAUSIBLE_RTT_MS:
return None # implausible: a broken measurement
if s.minrtt > s.srtt + RTT_INVERSION_TOLERANCE_MS:
return None # floor above the average: stale field
return (s.srtt, s.minrtt, max(0.0, s.srtt - s.minrtt))
def latency_stats(socks: dict) -> dict:
"""What the user's own TCP connections are experiencing, or None.
Pure and injected so it can be tested without a live socket table.
Returns None when too few connections qualify — an honest blank beats a
distribution invented from three sockets.
"""
srtts, floors, queues, retrans_socks, rejected = [], [], [], 0, 0
for s in socks.values():
if s.srtt is None or s.minrtt is None:
continue # not a rejection: the kernel simply has no timing
t = socket_timing(s)
if t is None:
rejected += 1
continue
srtts.append(t[0])
floors.append(t[1])
queues.append(t[2])
if s.retrans:
retrans_socks += 1
if len(srtts) < MIN_LATENCY_SOCKETS:
return None
srtts.sort(); floors.sort(); queues.sort()
return {
"sockets": len(srtts),
"rejected": rejected,
# What the applications see, end to end, including distance.
"rtt_p50": round(statistics.median(srtts), 2),
"rtt_p95": round(nearest_rank(srtts, 0.95), 2),
# The structural floor of the paths in use.
"floor_p50": round(statistics.median(floors), 2),
# Queueing, with distance divided out. This is the number that
# compares across connections.
"queue_p50": round(statistics.median(queues), 2),
"queue_p95": round(nearest_rank(queues, 0.95), 2),
"retrans_sockets": retrans_socks,
}
def parse_ss(raw: str, max_sockets: int = 10_000) -> dict:
"""{socket_key: (app, pid, sent, received)} from `ss -tinpH` output.
Sockets are keyed by local/peer address pair plus pid, which survives
across samples for the life of the connection. Sockets without process
attribution (other users' processes) are skipped, and parsing stops at
max_sockets so a pathological table cannot expand retained state.
"""
out = {}
addr = None
app = None
pid = None
for line in raw.splitlines():
if not line.startswith(("\t", " ")):
# Header line: state, queues, local, peer, users.
parts = line.split()
addr = None
app = None
m = RE_USER.search(line)
if m and len(parts) >= 5:
addr = parts[3] + ">" + parts[4]
app = m.group(1)
pid = int(m.group(2))
continue
if addr is None:
continue
sent = RE_SENT.search(line)
recv = RE_RECV.search(line)
if sent or recv:
if len(out) >= max_sockets:
break
# The kernel's own timing, off the same line. Absent on a socket
# it has not measured yet, which is why these stay None rather
# than defaulting to zero.
rtt = RE_RTT.search(line)
minrtt = RE_MINRTT.search(line)
retrans = RE_RETRANS.search(line)
out[addr + "#" + str(pid)] = Sock(
app, pid,
int(sent.group(1)) if sent else 0,
int(recv.group(1)) if recv else 0,
float(rtt.group(1)) if rtt else None,
float(minrtt.group(1)) if minrtt else None,
int(retrans.group(1)) if retrans else 0,
)
addr = None
return out
# Reaping is inside the budget, not after it. This whole call runs on the
# daemon's loop, so what the caller needs bounded is the block the loop
# suffers — the read plus getting rid of the child. Spending the deadline
# reading and then seconds terminating meets the letter and misses the
# point; that gap is what let a slow `ss` push live.json past the age at
# which the bar calls the daemon dead.
REAP_RESERVE_S = 0.25
def read_bounded(proc, max_bytes: int, deadline_s: float):
"""The child's stdout, capped in size and in time, then the child reaped.
Returns the text, or None if the process did not finish within the
deadline — it is killed and waited for either way, so nothing lingers.
Output past the cap is discarded and the process stopped: the sample
stays bounded and simply under-counts, which is the existing contract.
`deadline_s` bounds the CALL, reaping included, because that is the
figure the caller budgets against.
"""
fd = proc.stdout.fileno()
os.set_blocking(fd, False)
reserve = max(0.05, min(REAP_RESERVE_S, deadline_s * 0.5))
deadline = time.monotonic() + max(0.0, deadline_s - reserve)
chunks, total, timed_out = [], 0, False
try:
while total <= max_bytes:
remaining = deadline - time.monotonic()
if remaining <= 0:
timed_out = True
break
ready, _, _ = select.select([fd], [], [], remaining)
if not ready:
continue
try:
chunk = os.read(fd, min(65536, max_bytes + 1 - total))
except BlockingIOError:
continue
if not chunk:
break
chunks.append(chunk)
total += len(chunk)
except OSError:
timed_out = True
finally:
proc.stdout.close()
_reap(proc, reserve)
if timed_out:
return None
return b"".join(chunks)[:max_bytes].decode("utf-8", "replace")
def _reap(proc, budget_s: float):
"""Terminate if still running, escalate to kill, and wait — within
`budget_s`, because this runs on the daemon's loop.
A signalled child that is never waited for is a zombie until the next
Popen happens to collect it, so waiting is right; waiting without a
bound is not. A child that survives SIGKILL for longer than this is in
uninterruptible sleep, and no amount of further waiting is going to
help — leaving it for the next poll to collect costs a transient
zombie, while blocking here costs the snapshot the whole bar reads.
"""
half = max(0.05, budget_s * 0.5)
try:
if proc.poll() is None:
proc.terminate()
proc.wait(timeout=half)
except subprocess.TimeoutExpired:
try:
proc.kill()
proc.wait(timeout=half)
except (OSError, subprocess.TimeoutExpired):
pass
except OSError:
pass
class AppTraffic:
"""Aggregates socket samples into per-app rates and session totals."""
# How many poll intervals of history each app keeps. At the 3-second
# poll that is one minute — enough to show the shape of usage, small
# enough to ride along in apps.json.
HISTORY = 20
def __init__(self):
self.prev = {}
self.prev_t = None
self.totals = {} # app -> [rx_bytes, tx_bytes]
self.rates = [] # last interval's list, ready for apps.json
self.history = {} # app -> deque of [rx_bps, tx_bps]
self.latency = None # last latency_stats(), or None while under-sampled
# Enumeration bounds: a machine with an enormous socket table must not
# make the daemon allocate without limit every three seconds. 4 MB of
# `ss` output is roughly 8000 sockets — far past any laptop, and the
# cap degrades to "top apps among the first N sockets", not a crash.
MAX_SS_BYTES = 4 * 1024 * 1024
MAX_SOCKETS = 10_000
# `ss -p` walks every process's descriptors to name the owners, and
# on a busy machine that can stall. The read is bounded in time as
# well as size, because a read with no deadline holds the daemon's
# loop — and its outage watch — for as long as `ss` does.
#
# The size of the bound is not free either. This call is the only
# blocking one left on the loop, and the loop writes live.json just
# before it, so whatever this is budgeted for is how stale that
# snapshot can get. The bar calls the daemon dead at
# `BarWidget.staleAfterS` (5 s) — so a 5 s budget here, which is what
# this was, could blank the index, the headline and the path
# sparklines' liveness ring on a daemon that was measuring perfectly.
# A third of the contract leaves room for the rest of the iteration.
# Overrunning it costs one interval of app counters, which the Apps
# tab already degrades honestly; the alternative cost the whole bar.
# A test pins this against the QML number rather than either alone.
POLL_DEADLINE_S = 1.5
def poll(self) -> bool:
if not shutil.which("ss"):
return False
try:
proc = subprocess.Popen(["ss", "-tinpH"], stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL)
except OSError:
return False
raw = read_bounded(proc, self.MAX_SS_BYTES, self.POLL_DEADLINE_S)
if raw is None:
return False
now = time.time()
cur = parse_ss(raw, max_sockets=self.MAX_SOCKETS)
self._fold(cur, now)
return True
def _fold(self, cur: dict, now: float):
if self.prev_t is None:
# First sample is the baseline: the counters carry each
# connection's whole history, which is not this interval's
# traffic.
self.prev, self.prev_t = cur, now
return
dt = max(0.5, now - self.prev_t)
interval = {} # app -> [rx, tx, conns]
per_app_rtt = {} # app -> list of srtt
per_app_queue = {} # app -> list of srtt - minrtt
for key, s in cur.items():
app, sent, recv = s.app, s.sent, s.recv
prev = self.prev.get(key)
if prev is not None:
d_tx = max(0, sent - prev[2])
d_rx = max(0, recv - prev[3])
else:
# Born since the last sample: its whole life is this
# interval.
d_tx, d_rx = sent, recv
slot = interval.setdefault(app, [0, 0, 0])
slot[0] += d_rx
slot[1] += d_tx
slot[2] += 1
# Per-app timing shares the aggregate's guard, so a socket
# rejected there cannot be silently counted here.
t = socket_timing(s)
if t is not None:
per_app_rtt.setdefault(app, []).append(t[0])
per_app_queue.setdefault(app, []).append(t[2])
# Sockets that closed between samples take their final delta with
# them — the tail of a closed connection is the one thing this
# method genuinely cannot count.
for app, (rx, tx, _conns) in interval.items():
tot = self.totals.setdefault(app, [0, 0])
tot[0] += rx
tot[1] += tx
# Every known app gets a history sample each interval — an app that
# went quiet records zeros, so its strip shows the quiet.
for app in self.totals:
v = interval.get(app)
h = self.history.setdefault(app, deque(maxlen=self.HISTORY))
h.append([round(v[0] / dt, 1), round(v[1] / dt, 1)] if v else [0.0, 0.0])
self.rates = [
{
"name": app,
"rx_bps": round(v[0] / dt, 1),
"tx_bps": round(v[1] / dt, 1),
"conns": v[2],
"rx_total": self.totals.get(app, [0, 0])[0],
"tx_total": self.totals.get(app, [0, 0])[1],
"hist": list(self.history.get(app, [])),
# Median across this app's qualifying sockets. None when it
# has none — an app talking only QUIC shows no latency here,
# which is honest rather than zero.
"rtt_ms": (round(statistics.median(per_app_rtt[app]), 2)
if per_app_rtt.get(app) else None),
"queue_ms": (round(statistics.median(per_app_queue[app]), 2)
if per_app_queue.get(app) else None),
}
for app, v in interval.items()
]
self.latency = latency_stats(cur)
self.prev, self.prev_t = cur, now
def top(self, n: int = 8) -> list:
"""Busiest apps first; idle-but-heavy session users still listed."""
ranked = sorted(self.rates,
key=lambda a: (a["rx_bps"] + a["tx_bps"],
a["rx_total"] + a["tx_total"]),
reverse=True)
seen = {a["name"] for a in ranked}
# Apps with session history but no sockets this interval.
idle = [
{"name": app, "rx_bps": 0.0, "tx_bps": 0.0, "conns": 0,
"rx_total": t[0], "tx_total": t[1],
"hist": list(self.history.get(app, [])),
"rtt_ms": None, "queue_ms": None}
for app, t in self.totals.items() if app not in seen
]
idle.sort(key=lambda a: a["rx_total"] + a["tx_total"], reverse=True)
return (ranked + idle)[:n]
plugins/io.github.x3me.nexthop/nexthopd/cli.py
+365
View File
@@ -0,0 +1,365 @@
"""`nexthop` — the query CLI the panel (and you) use for history.
Everything answers in JSON on stdout, because the consumer is a QML
Process { } as often as it is a person. The daemon is not involved: reads go
straight to the files — sqlite for history (WAL mode makes that safe), the
JSON state files for the rest.
nexthop query --window 24h history series at the right resolution
nexthop live the current live.json
nexthop events --window 7d outages, disruptions, changes
nexthop tests [--kind peak] speed test results
nexthop report --window 24h plain-text summary for an ISP ticket
nexthop peak ask the running daemon for a peak test
nexthop stream <keys> the shell's bounded reader: one JSON line
per named state file, re-emitted on change
nexthop retire --pid --start SIGTERM a stale daemon, identity-checked
"""
import argparse
import fcntl
import json
import os
import signal
import stat
import sys
import time
from .paths import (apps_path, db_path, live_path, lock_path, manifest_path,
recent_path)
from .state import read_json, read_text_bounded
WINDOWS = {"m": 60, "h": 3600, "d": 86400}
def parse_window(text: str) -> float:
text = (text or "30m").strip().lower()
unit = text[-1]
if unit in WINDOWS:
try:
return float(text[:-1]) * WINDOWS[unit]
except ValueError:
pass
try:
return float(text)
except ValueError:
return 1800.0
def emit(obj):
json.dump(obj, sys.stdout, separators=(",", ":"))
print()
def cmd_live(_args):
emit(read_json(live_path(), {"state": "no-daemon"}))
return 0
# The QML side never opens a state file itself. It runs `nexthop stream`
# and reads whole lines, so the only code that touches these paths is the
# bounded no-follow non-blocking read in state.py — an oversized file, a
# FIFO or a symlink swap is refused here, in a small short-lived process,
# instead of allocating or stalling inside the long-lived shell.
#
# Keys, never paths: the caller picks from this table, so no argument it
# passes can widen what gets opened. Caps match each file's real size
# (live ~3 KB, apps ~8 KB, recent ~30 KB) with generous headroom.
STREAMABLE = {
"live": (live_path, 256 * 1024),
"apps": (apps_path, 1024 * 1024),
"recent": (recent_path, 1024 * 1024),
"manifest": (manifest_path, 256 * 1024),
}
def cmd_stream(args):
"""Emit `<key> <json>` lines whenever a watched file's contents change.
The payload is re-serialised here rather than forwarded verbatim: it
guarantees one line per record whatever the file's own formatting
(manifest.json is indented, the state files are not), and it means
only JSON this process already parsed successfully is ever handed to
the shell.
"""
keys = [k for k in dict.fromkeys(args.keys) if k in STREAMABLE]
if not keys:
print("stream: nothing to watch", file=sys.stderr)
return 2
interval = min(max(args.interval, 0.1), 60.0)
last = {}
while True:
for key in keys:
resolve, cap = STREAMABLE[key]
got = read_text_bounded(resolve(), cap)
if got is None:
continue
text, stamp = got
if last.get(key) == stamp:
continue
last[key] = stamp
try:
payload = json.loads(text)
except ValueError:
continue # a half-written or foreign file; skip it
# ensure_ascii escapes any newline inside a string, so the
# record cannot break the line framing.
line = json.dumps(payload, separators=(",", ":"))
try:
sys.stdout.write(f"{key} {line}\n")
sys.stdout.flush()
except (BrokenPipeError, ValueError):
# The shell went away; so do we. _exit skips the
# interpreter's final flush, which would only raise the
# same broken pipe again and print it to stderr.
os._exit(0)
time.sleep(interval)
def authorized_to_retire(pid: int, want_start: int) -> bool:
"""Is this pid really our daemon, and the same one live.json named?
Three independent facts, all read from /proc and none of them a name
match: the process must belong to this user, its argv must be exactly
a python interpreter running `-m nexthopd`, and its start time must
equal the one the daemon published. A recycled pid can reproduce the
number but never the start time.
This lives here rather than in a shell one-liner because the one-liner
could not be tested and, as it turned out, did not run at all: the
NUL it passed to `tr` truncated the script at execve.
"""
try:
if os.stat(f"/proc/{pid}").st_uid != os.getuid():
return False
with open(f"/proc/{pid}/cmdline", "rb") as f:
argv = [a.decode("utf-8", "replace")
for a in f.read(4096).split(b"\0") if a]
with open(f"/proc/{pid}/stat", "rb") as f:
data = f.read(4096)
except (OSError, ValueError):
return False
if len(argv) < 3 or "python" not in os.path.basename(argv[0]):
return False
if argv[1] != "-m" or argv[2] != "nexthopd" or len(argv) > 3:
return False
if want_start:
try:
start = int(data[data.rindex(b")") + 2:].split()[19])
except (ValueError, IndexError):
return False
if start != want_start:
return False
return True
def cmd_retire(args):
"""SIGTERM a stale daemon, but only once its identity checks out."""
if args.pid <= 0 or args.pid == os.getpid():
return 1
if not authorized_to_retire(args.pid, args.start):
return 1
try:
os.kill(args.pid, signal.SIGTERM)
except OSError:
return 1
return 0
def open_store():
# Imported here, not at module scope: `stream` runs for the life of the
# shell and has no use for sqlite3, so it should not pay to load it.
from .store import Store
path = db_path()
if not path.exists():
return None
try:
return Store(path, read_only=True)
except Exception:
return None
def cmd_query(args):
store = open_store()
if not store:
emit({"error": "no history yet"})
return 1
seconds = parse_window(args.window)
rows, table = store.series(seconds, resolution=args.resolution)
emit({"window_s": seconds, "resolution": table, "rows": rows})
return 0
def cmd_events(args):
store = open_store()
if not store:
emit({"events": []})
return 0
emit({"events": store.events(parse_window(args.window))})
return 0
def cmd_tests(args):
store = open_store()
if not store:
emit({"tests": []})
return 0
emit({"tests": store.tests(limit=args.limit, kind=args.kind)})
return 0
def _lock_holder_pid() -> int:
"""The pid written by whoever holds the daemon lock, or 0.
The lock file outlives a daemon that died hard and pids are recycled,
so the number in the file is not evidence on its own. The flock is:
if this process can take it, nobody holds it and nobody is listening,
whatever the file says. Opened the way the daemon opens it — no
symlink following, a regular file or nothing — and never truncated.
"""
try:
fd = os.open(lock_path(), os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC)
except OSError:
return 0
try:
if not stat.S_ISREG(os.fstat(fd).st_mode):
return 0
try:
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
# Held: that is the daemon, and the pid it wrote after taking
# the lock is the one to ring.
try:
return int(os.read(fd, 32).strip())
except ValueError:
return 0
# We got it, so nobody was holding it. Give it straight back.
fcntl.flock(fd, fcntl.LOCK_UN)
return 0
except OSError:
return 0
finally:
os.close(fd)
def cmd_peak(_args):
"""Ring the daemon's doorbell. SIGUSR1 is the whole protocol.
SIGUSR1's default disposition is *terminate*, so signalling the wrong
pid is not a harmless no-op — and this is what a middle-click on the
bar runs. Two checks stand before the signal: the lock must actually
be held (see `_lock_holder_pid`), and the holder must pass the same
owner-and-argv test the retire path applies, with the start time
live.json published when it names the same pid. The same rule as
`nexthop retire`: a process is authorised by identity, never by the
number a file happens to contain.
"""
pid = _lock_holder_pid()
if pid <= 0 or pid == os.getpid():
emit({"ok": False, "error": "daemon not running"})
return 1
live = read_json(live_path(), {}) or {}
want_start = 0
if live.get("pid") == pid:
try:
want_start = int(live.get("pid_start") or 0)
except (TypeError, ValueError):
want_start = 0
if not authorized_to_retire(pid, want_start):
emit({"ok": False, "error": "lock holder is not nexthopd"})
return 1
try:
os.kill(pid, signal.SIGUSR1)
except OSError:
emit({"ok": False, "error": "daemon not running"})
return 1
emit({"ok": True})
return 0
def cmd_report(args):
"""The paste-into-a-ticket summary. Plain text by design."""
store = open_store()
live = read_json(live_path(), {})
seconds = parse_window(args.window)
lines = []
link = live.get("link", {})
lines.append(f"Nexthop report — last {args.window}")
lines.append(f"generated {time.strftime('%Y-%m-%d %H:%M %Z')}")
if link:
what = link.get("ssid") or link.get("name") or link.get("iface", "?")
lines.append(f"connection: {what} ({link.get('kind', '?')}), "
f"gateway {link.get('gateway', '?')}")
lines.append("")
if store:
rows, table = store.series(seconds)
vals = lambda k: [r[k] for r in rows if r.get(k) is not None]
def block(name, p50key, p95key, losskey):
p50, p95, loss = vals(p50key), vals(p95key), vals(losskey)
if not p50:
lines.append(f"{name}: no data")
return
lines.append(
f"{name}: median {sum(p50)/len(p50):.1f} ms, "
f"p95 {max(p95) if p95 else 0:.1f} ms (worst {table} bucket), "
f"loss {sum(loss)/len(loss)*100 if loss else 0:.2f}%")
block("local leg (to router)", "local_p50", "local_p95", "local_loss")
block("wan leg (past router)", "wan_p50", "wan_p95", "wan_loss")
lines.append("")
events = store.events(seconds)
if events:
lines.append("events:")
for e in events:
start = time.strftime("%a %H:%M", time.localtime(e["ts"]))
dur = (f"{e['ended_ts'] - e['ts']}s" if e["ended_ts"]
else "ongoing")
lines.append(f" {start} {e['kind']} on {e['leg']} leg, {dur}"
f" — {e['detail']}")
else:
lines.append("events: none")
tests = store.tests(limit=5)
if tests:
lines.append("")
lines.append("speed tests:")
for t in tests:
when = time.strftime("%a %H:%M", time.localtime(t["ts"]))
down = f"{t['down_mbps']:.0f}" if t["down_mbps"] else "--"
up = f"{t['up_mbps']:.0f}" if t["up_mbps"] else "--"
lines.append(f" {when} {t['kind']:<8} {down}/{up} Mbps"
f" ({t['engine']})")
print("\n".join(lines))
return 0
def main(argv=None):
ap = argparse.ArgumentParser(prog="nexthop")
sub = ap.add_subparsers(dest="cmd", required=True)
sub.add_parser("live")
q = sub.add_parser("query")
q.add_argument("--window", default="30m")
q.add_argument("--resolution", default="auto",
choices=["auto", "minute", "hour"])
e = sub.add_parser("events")
e.add_argument("--window", default="7d")
t = sub.add_parser("tests")
t.add_argument("--kind", default=None)
t.add_argument("--limit", type=int, default=20)
sub.add_parser("peak")
r = sub.add_parser("report")
r.add_argument("--window", default="24h")
s = sub.add_parser("stream")
s.add_argument("keys", nargs="+", choices=sorted(STREAMABLE))
s.add_argument("--interval", type=float, default=0.5)
rt = sub.add_parser("retire")
rt.add_argument("--pid", type=int, required=True)
rt.add_argument("--start", type=int, default=0)
args = ap.parse_args(argv)
return {"live": cmd_live, "query": cmd_query, "events": cmd_events,
"tests": cmd_tests, "peak": cmd_peak, "report": cmd_report,
"stream": cmd_stream, "retire": cmd_retire}[args.cmd](args)
if __name__ == "__main__":
sys.exit(main())
plugins/io.github.x3me.nexthop/nexthopd/daemon.py
+2056
View File
File diff suppressed because it is too large Load Diff
plugins/io.github.x3me.nexthop/nexthopd/instruments.py
+353
View File
@@ -0,0 +1,353 @@
"""Two best instruments for the internet leg.
One anchor is one opinion. A single probe target that rate-limits, gets
DDoSed, or sits behind a bad peering path poisons the score for everyone
behind it — and there is no way to tell "my internet is slow" from "the
anchor is having a day" with one instrument. Orb's answer, adopted here:
keep a small pool of instruments, score from the best two, re-rank them
continuously, and never let one flapping target churn the pair.
An instrument is a (protocol, target) pair. The pool mixes protocols on
purpose: TCP handshakes travel the application path, ICMP is the cheapest
edge detector — each can fail alone, and the pair means neither failing
alone moves the score. Ranking is arguable-by-design, one number at a
time, like the score anchors:
penalty = 2000·loss + (p95 − p50) + 0.1·p50
Loss dominates (5 % of packets lost costs as much as 100 ms of tail spread), the
tail spread comes second because Lag leans on p75, and the median is a
tiebreak — an instrument must not win its seat merely by being close.
Damping, because re-selection is where naive versions of this oscillate:
a challenger must beat the worst active instrument by 20 % on two
consecutive evaluations; an instrument whose seat changes three or more
times in an hour is quarantined for thirty minutes; and a dead active
instrument is replaced immediately, hysteresis notwithstanding — waiting
two rounds to bench a corpse helps nobody.
Everything here is pure bookkeeping over injected stats. No probe, no
subprocess, no clock of its own — which is what makes it testable, and
tested.
"""
import statistics
from collections import deque
from .probes import RECENT_MIN_SAMPLES, RECENT_WINDOW_S, Series
DEAD_PENALTY = 2000.0 # loss = 1.0 and nothing else to say
def penalty(stats):
"""Rank one instrument's recent window; None = not enough to judge."""
if not stats:
return None
count = stats.get("count") or 0
if count < Bench.MIN_SAMPLES:
return None
loss = stats.get("loss") or 0.0
p50, p95 = stats.get("p50"), stats.get("p95")
if p50 is None:
return DEAD_PENALTY * loss if loss > 0 else None
spread = (p95 - p50) if p95 is not None else 0.0
return DEAD_PENALTY * loss + spread + 0.1 * p50
def host_of(target: str) -> str:
"""The address part of an instrument's target.
So that two instruments pointed at the same machine are recognisable as
such: `1.1.1.1` and `1.1.1.1:443` are one host wearing two protocols. A
trailing `:port` is stripped only when what remains holds no colon of its
own, which leaves an IPv6 literal intact rather than truncating it.
"""
head, sep, tail = target.rpartition(":")
if sep and tail.isdigit() and ":" not in head:
return head
return target
class Instrument:
def __init__(self, key: str, kind: str, target: str = ""):
self.key = key
self.kind = kind # "icmp" | "tcp"
self.target = target
self.active = False
self.pending_wins = 0 # consecutive evaluations won as challenger
self.seat_changes = deque(maxlen=32) # timestamps, for flap tracking
self.quarantined_until = 0.0
def flapping(self, now, window_s, limit) -> bool:
return sum(1 for t in self.seat_changes if now - t <= window_s) >= limit
class Bench:
"""Holds the pool, decides who sits in the two scored seats."""
ACTIVE_N = 2
WINDOW_S = RECENT_WINDOW_S # ranking window, shared with TcpProbe
RESELECT_EVERY_S = 300.0 # ordinary re-ranking cadence
MIN_SAMPLES = RECENT_MIN_SAMPLES # below this a window judges nothing
MARGIN = 0.8 # challenger must be 20% better than the seat
CONSECUTIVE_WINS = 2
FLAP_WINDOW_S = 3600.0
FLAP_LIMIT = 3
QUARANTINE_S = 1800.0
# An active instrument at or past this penalty is not "worse", it is
# gone — full loss, or no samples arriving at all.
DEAD_AT = DEAD_PENALTY * 0.95
def __init__(self, pool):
"""pool: ordered [(key, kind, target)].
The opening seats span two DISTINCT hosts rather than being the first
two in the pool. They used to be `pool[:2]` — ICMP and TCP to the
anchor, the pre-0.2.0 pair, kept for continuity until the first
ranking. On any network that blocks the anchor outright that put both
scored seats on a dead host at every start: the outage watch opens at
4 s, the notification fires at 5 s, and the bench cannot reseat until
its next pass a minute later. A false outage and a desktop alert on
every daemon start, and the daemon restarts on a shell restart, a
version handover and a probe-settings change (#5).
One working seat is enough to prevent it — the leg answers if either
instrument does — so the rule is simply that the pair must not be one
host twice. The bench re-ranks from there as it always did; this only
decides what is seated before there is anything to rank.
Not fixed by changing the default anchor: every candidate address is
blocked on somebody's network, so that moves the report rather than
closing it.
"""
self.instruments = {}
for key, kind, target in pool:
self.instruments[key] = Instrument(key, kind, target)
for inst in self._opening_seats():
inst.active = True
self._last_reselect = 0.0
def _opening_seats(self):
"""The first instruments of the pool that do not share a host."""
seats, hosts = [], set()
for inst in self.instruments.values():
if len(seats) >= self.ACTIVE_N:
break
host = host_of(inst.target)
if host in hosts:
continue
seats.append(inst)
hosts.add(host)
# A pool offering fewer distinct hosts than there are seats fills the
# rest in order: fewer scored instruments than the bench expects is a
# worse failure than two of them sharing a host.
if len(seats) < self.ACTIVE_N:
for inst in self.instruments.values():
if len(seats) >= self.ACTIVE_N:
break
if inst not in seats:
seats.append(inst)
return seats
def actives(self):
return [i for i in self.instruments.values() if i.active]
def _healthy(self, pens, inst, now):
p = pens.get(inst.key)
return (p is not None and p < self.DEAD_AT
and now >= inst.quarantined_until)
def _seat(self, inst, now, active: bool):
if inst.active == active:
return None
inst.active = active
inst.pending_wins = 0
inst.seat_changes.append(now)
if inst.flapping(now, self.FLAP_WINDOW_S, self.FLAP_LIMIT):
inst.quarantined_until = now + self.QUARANTINE_S
return (inst.key, active)
def evaluate(self, now, stats_by_key):
"""One pass; returns [(key, now_active)] seat changes.
Call every minute or so: emergency replacement of a dead seat acts
on any pass, ordinary re-ranking only every RESELECT_EVERY_S.
"""
pens = {k: penalty(stats_by_key.get(k)) for k in self.instruments}
changes = []
# A dead seat is replaced now. No hysteresis for corpses — but no
# churn during a full outage either: promotion needs a healthy
# standby, and when everything is dead the pair stands still.
for inst in self.actives():
p = pens.get(inst.key)
if p is not None and p < self.DEAD_AT:
continue
standbys = [i for i in self.instruments.values()
if not i.active and self._healthy(pens, i, now)]
if not standbys:
continue
best = min(standbys, key=lambda i: pens[i.key])
changes += filter(None, [self._seat(inst, now, False),
self._seat(best, now, True)])
if now - self._last_reselect < self.RESELECT_EVERY_S:
return changes
self._last_reselect = now
# Ordinary re-ranking, damped. The worst seat defends against the
# best healthy challenger; a challenger that stops winning starts
# over from zero.
actives = [i for i in self.actives() if pens.get(i.key) is not None]
challengers = [i for i in self.instruments.values()
if not i.active and self._healthy(pens, i, now)]
for inst in self.instruments.values():
if not inst.active and inst not in challengers:
inst.pending_wins = 0
if not actives or not challengers:
return changes
worst = max(actives, key=lambda i: pens[i.key])
best = min(challengers, key=lambda i: pens[i.key])
for c in challengers:
if c is not best:
c.pending_wins = 0
if pens[best.key] < pens[worst.key] * self.MARGIN:
best.pending_wins += 1
if best.pending_wins >= self.CONSECUTIVE_WINS:
changes += filter(None, [self._seat(worst, now, False),
self._seat(best, now, True)])
else:
best.pending_wins = 0
return changes
def snapshot(self, now, stats_by_key):
"""For live.json: who is in the pool, who holds a seat, and how
each has been measuring — so the shell can show the bench."""
out = []
for inst in self.instruments.values():
st = stats_by_key.get(inst.key) or {}
out.append({
"key": inst.key, "kind": inst.kind, "target": inst.target,
"active": inst.active,
"quarantined": now < inst.quarantined_until,
"p50": st.get("p50"), "p95": st.get("p95"),
# Per instrument, so the field can show what the merged
# figure used to hide: jitter measured within one stream.
"jitter": st.get("jitter"),
"loss": st.get("loss"), "count": st.get("count") or 0,
})
return out
def merged_stats(sample_lists) -> dict:
"""Series.stats over several instruments, each counting once.
Pooling the seated instruments' raw samples and calling Series.stats on
the pile — what this replaces — got two things wrong, and both moved
the score:
* Jitter is RFC 3550 IPDV, the difference between consecutive replies,
and consecutive replies in a pooled stream come from different
instruments. Two perfectly stable instruments with different base
round trips read as jittery: replayed at this line's own figures
(ICMP 3.41 ms at 500 ms, TCP 4.82 ms at 1 s) the pool reported
0.93 ms of jitter from two streams with none, and on a router that
fast-paths ICMP (5 vs 15 ms) it reported 6.6 ms and took ten points
off Responsiveness for nothing.
* Every instrument was weighted by how often it happened to probe.
ICMP follows the probeIntervalMs setting and TCP is fixed at one a
second, so changing a setting changed p75, loss and the index while
the network stayed the same: the 5/15 case scored 100, 92.7 or 90.3
depending only on that number.
Here each instrument's replies carry weight 1/n, percentiles are the
weighted nearest rank over the pool, jitter is the mean of the
instruments' own IPDVs, and loss is the mean of their loss rates.
One instrument reduces to Series.stats exactly, so `lag_icmp` and the
local leg are untouched.
"""
lists = [lst for lst in sample_lists if lst]
if not lists:
return Series.stats([])
if len(lists) == 1:
return Series.stats(lists[0])
per = [Series.stats(lst) for lst in lists]
count = sum(p["count"] for p in per)
loss = statistics.fmean(p["loss"] for p in per)
replies = [[smp[1] for smp in lst if smp[1] is not None] for lst in lists]
voiced = [r for r in replies if r]
if not voiced:
return {"count": count, "loss": loss, "p50": None, "p75": None,
"p95": None, "jitter": None, "last": None, "max": None}
pooled = []
for r in voiced:
w = 1.0 / len(r)
pooled.extend((v, w) for v in r)
pooled.sort(key=lambda x: x[0])
total_w = float(len(voiced))
def pct(p):
target = p * total_w - 1e-9
cum = 0.0
for v, w in pooled:
cum += w
if cum >= target:
return v
return pooled[-1][0]
jitters = [p["jitter"] for p in per if p["jitter"] is not None]
newest = None
for lst in lists:
for smp in reversed(lst):
if smp[1] is not None:
if newest is None or smp[0] > newest[0]:
newest = smp
break
return {
"count": count,
"loss": loss,
"p50": round(pct(0.5), 2),
"p75": round(pct(0.75), 2),
"p95": round(pct(0.95), 2),
"max": round(pooled[-1][0], 2),
"jitter": round(statistics.fmean(jitters), 2) if jitters else 0.0,
"last": round(newest[1], 2) if newest else None,
}
class MergedSeries:
"""A read-only view over whichever instruments hold the seats.
`.since()` and `.all()` return the pooled stream in time order — right
for anything that asks "did anyone reply between these two moments",
which is what outage detection does. Anything that turns the leg into
statistics goes through `.stats()` / `.each()` and `merged_stats`,
where the instruments count equally; see that function for why the
pooled stream must not be fed to Series.stats.
"""
def __init__(self, series_fn):
self._series_fn = series_fn # -> [Series] of the active seats
def each(self, seconds: float = None):
"""One sample list per seated instrument, the shape merged_stats wants."""
if seconds is None:
return [s.all() for s in self._series_fn()]
return [s.since(seconds) for s in self._series_fn()]
def stats(self, seconds: float) -> dict:
return merged_stats(self.each(seconds))
def since(self, seconds: float):
out = []
for s in self._series_fn():
out.extend(s.since(seconds))
out.sort(key=lambda smp: smp[0])
return out
def all(self):
out = []
for s in self._series_fn():
out.extend(s.all())
out.sort(key=lambda smp: smp[0])
return out
plugins/io.github.x3me.nexthop/nexthopd/linkevents.py
+194
View File
@@ -0,0 +1,194 @@
"""Who ended the last Wi-Fi association: the access point, this machine,
or nobody (the link fell over).
A BSSID change seen through `iw dev link` is only a fact: we were on one
radio and now we are on another. Whether that was the client's decision,
the access point's, or a link failure is the difference between a laptop
that roams, a router that kicks, and a driver that drops — three problems
with three owners, and the log used to call all of them "Roamed to".
`iw event` is the unprivileged view of nl80211's mlme multicast group.
Every deauthentication and disassociation frame the kernel sends or
receives is reported with its sender, receiver and 802.11 reason code, and
the authentication that follows is reported too. That is enough:
- the frame's sender says who ended it — the AP's address means it kicked
us, our own means this machine did;
- the reason code says why, in the AP's or the driver's own words;
- the delay before the next authentication says whether the client already
knew where it was going. mac80211 emits the local deauth from inside the
call that starts the new authentication when it roams, so a roam's gap is
milliseconds; a lost link is followed by a scan first, and its gap is
seconds.
Only the kernel talks to this module, through `iw`. Lines are read with a
length cap and matched against fixed patterns; two MACs and a reason code
are the only fields kept, and only our own reason texts reach the shell.
"""
import re
import shutil
import subprocess
import threading
import time
from collections import deque
# `iw event -t` line: "<secs>.<usecs>: <ifname> (phy #N): <event...>". The
# timestamp is CLOCK_REALTIME — the clock the daemon stamps with — and it
# is optional so recorded fixtures read the same with or without it.
_PREFIX = r"^(?:(\d+\.\d+): )?\S+(?: \(phy #\d+\))?: "
_MAC = r"([0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5})"
_ANY_MAC = r"[0-9a-fA-F]{2}(?::[0-9a-fA-F]{2}){5}"
# A deauth/disassoc frame: "<sender> -> <receiver> reason N: <text>".
# "unprotected deauth" (a forged frame the kernel refused) does not match:
# the frame word must follow the prefix directly.
RE_FRAME = re.compile(_PREFIX + r"(deauth|disassoc) " + _MAC + " -> " + _MAC
+ r" reason (\d{1,5})\b")
# The first sign of the next association landing.
RE_NEXT = re.compile(_PREFIX + r"(?:(?:auth|assoc) " + _ANY_MAC + " -> "
+ _ANY_MAC + r" status: 0\b|(?:connected|roamed) to "
+ _ANY_MAC + r"\b)")
# 802.11 reason codes, in the words a person would use. Codes 3 and 4 mean
# different things depending on who sent them, so they get a side each
# (a GX gateway's per-station kick arrives as 8, for instance).
REASONS = {
1: "unspecified",
2: "previous authentication no longer valid",
5: "the AP is full",
6: "class 2 frame from an unauthenticated station",
7: "class 3 frame from an unassociated station",
9: "not authenticated",
14: "MIC failure",
15: "4-way handshake timeout",
16: "group key handshake timeout",
17: "IE mismatch in the 4-way handshake",
23: "802.1X authentication failed",
34: "poor channel conditions",
39: "timeout",
}
AP_REASONS = {3: "the AP is leaving", 4: "inactivity", 8: "the AP is leaving the BSS"}
LOCAL_REASONS = {3: "leaving", 4: "beacon loss", 8: "leaving the BSS"}
def reason_text(code, by_ap):
"""'reason 2: previous authentication no longer valid'. Only our own
words, never the tool's: nothing from the wire reaches the shell."""
side = AP_REASONS if by_ap else LOCAL_REASONS
text = side.get(code) or REASONS.get(code)
return "reason %d: %s" % (code, text) if text else "reason %d" % code
class NlEvents(threading.Thread):
"""Runs `iw event` forever, restarting it if it dies, keeping the last
few deauth/disassoc frames and when the next authentication followed.
Like the probes, this never raises into the daemon: without `iw`, or
when the socket is refused, the thread backs off and `cause_for`
answers None — and the link log falls back to plain "Roamed to".
"""
MAX_CAUSES = 64
LINE_CAP = 1024
def __init__(self):
super().__init__(name="nl-events", daemon=True)
self._stop = threading.Event()
self._proc = None
self._lock = threading.Lock()
self._causes = deque(maxlen=self.MAX_CAUSES)
def stop(self):
self._stop.set()
proc = self._proc
if proc and proc.poll() is None:
try:
proc.terminate()
except OSError:
pass
def run(self):
backoff = 1.0
while not self._stop.is_set():
if not shutil.which("iw"):
time.sleep(min(backoff, 30.0))
backoff = min(backoff * 2, 30.0)
continue
try:
self._run_once()
except Exception:
pass # a spawn or parse failure is a retry, not a crash
if not self._stop.is_set():
# `iw event` never exits on its own; if it did, nl80211
# refused us or the tool is broken — do not spin on it.
time.sleep(min(backoff, 30.0))
backoff = min(backoff * 2, 30.0)
def _run_once(self):
self._proc = subprocess.Popen(
["iw", "event", "-t"], stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL, text=True, errors="replace", bufsize=1,
)
try:
while not self._stop.is_set():
# A capped read: an over-long line comes back in pieces
# that match nothing, instead of growing a buffer.
line = self._proc.stdout.readline(self.LINE_CAP)
if not line:
break
self.consume(line)
finally:
proc, self._proc = self._proc, None
if proc:
try:
proc.terminate()
proc.wait(timeout=2)
except subprocess.TimeoutExpired:
# Would not go quietly: do not leave it running.
try:
proc.kill()
proc.wait(timeout=2)
except (OSError, subprocess.TimeoutExpired):
pass
except OSError:
pass
def consume(self, line: str, now: float = None):
m = RE_FRAME.match(line)
if m:
t = float(m.group(1)) if m.group(1) else (now or time.time())
with self._lock:
self._causes.append({
"t": t, "frame": m.group(2),
"sa": m.group(3).lower(), "da": m.group(4).lower(),
"reason": min(int(m.group(5)), 65535), "next_at": None,
})
return
m = RE_NEXT.match(line)
if m:
t = float(m.group(1)) if m.group(1) else (now or time.time())
with self._lock:
# Only the newest cause is still waiting for its follow-up.
if self._causes and self._causes[-1]["next_at"] is None:
self._causes[-1]["next_at"] = t
def cause_for(self, bssid: str, now: float, window: float):
"""The latest frame within `window` seconds that ended our
association with `bssid`, or None.
by_ap: the AP sent it. gap_s: seconds until the next authentication
was seen, None if none has been yet.
"""
bssid = (bssid or "").lower()
if not bssid:
return None
with self._lock:
for c in reversed(self._causes):
if now - c["t"] > window:
break
if bssid not in (c["sa"], c["da"]):
continue
gap = None if c["next_at"] is None else max(0.0, c["next_at"] - c["t"])
return {"t": c["t"], "frame": c["frame"], "by_ap": c["sa"] == bssid,
"reason": c["reason"], "gap_s": gap}
return None
plugins/io.github.x3me.nexthop/nexthopd/net.py
+372
View File
@@ -0,0 +1,372 @@
"""Reading the local end of the connection: route, interface, Wi-Fi link.
Everything here is a cheap read of /sys or a short-lived `ip` / `iw` call.
Nothing in this module blocks for longer than its subprocess timeout, and
every function degrades to None or {} rather than raising, because a laptop
that just suspended will fail all of them at once.
"""
import json
import ipaddress
import re
import shutil
import subprocess
import time
from typing import Optional
def _run(cmd, timeout=2.0) -> Optional[str]:
if not shutil.which(cmd[0]):
return None
try:
out = subprocess.run(
cmd, capture_output=True, text=True, timeout=timeout, check=False
)
except (subprocess.TimeoutExpired, OSError):
return None
return out.stdout if out.returncode == 0 else None
# Gateway ranges that phone and desktop tethering hand out. Each is fixed by
# its vendor and documented, so this is a table lookup with no network call
# and nothing to keep up to date. It is the only reliable signal available:
# iOS randomises both the hotspot BSSID and the gateway's hardware address
# (checked on a live hotspot — `66:f8:f9:…` and `a2:ee:1a:…`, both with the
# locally-administered bit set), so vendor lookup on either is useless, and
# NetworkManager reports such a connection as "no (guessed)" rather than
# metered.
TETHER_RANGES = (
# iOS Personal Hotspot, over Wi-Fi or USB. A /28 with the phone at .1.
("172.20.10.0/28", "ios", "iPhone"),
# Android Wi-Fi tethering, and its USB counterpart.
("192.168.43.0/24", "android", "Phone"),
("192.168.42.0/24", "android", "Phone"),
# Windows mobile hotspot.
("192.168.137.0/24", "windows", "Hotspot"),
)
def tether_from_gateway(gateway: str):
"""Is this gateway a phone sharing its connection? Pure, so it is tested.
Returns `{"kind", "label"}` or None. `label` is what to call the middle
node of the path — it is a phone, not a router, and drawing a router
there quietly mislabels both legs.
"""
if not gateway:
return None
try:
addr = ipaddress.ip_address(gateway)
except ValueError:
return None
for cidr, kind, label in TETHER_RANGES:
try:
if addr in ipaddress.ip_network(cidr):
return {"kind": kind, "label": label}
except ValueError:
continue
return None
def nm_metered(iface: str) -> bool:
"""Has the user explicitly marked this connection metered?
Only an explicit answer counts. NetworkManager guesses by default and
guesses wrong on a phone hotspot — a live one reports
`no (guessed)` — so a guess is treated as no answer at all rather than
as evidence either way.
"""
if not iface:
return False
# No binary check here: _run already answers None when nmcli is absent,
# and a second check in front of it kept the parser out of reach of the
# tests on a machine without NetworkManager — which is what CI is.
raw = _run(["nmcli", "-t", "-f", "GENERAL.METERED", "dev", "show", iface],
timeout=4.0)
if not raw:
return False
value = raw.split(":", 1)[-1].strip().lower() if ":" in raw else ""
return value.startswith("yes") and "guess" not in value
def route_to(anchor: str = "1.1.1.1") -> dict:
"""The interface, gateway and source address used to reach the anchor.
This is the single source of truth for "which connection am I on" — the
gateway it returns is the router leg's ping target.
"""
raw = _run(["ip", "-j", "route", "get", anchor])
if not raw:
return {}
try:
rows = json.loads(raw)
except ValueError:
return {}
if not rows:
return {}
r = rows[0]
return {
"iface": r.get("dev") or "",
"gateway": r.get("gateway") or "",
"src": r.get("prefsrc") or "",
}
def is_wireless(iface: str) -> bool:
if not iface:
return False
from pathlib import Path
return Path(f"/sys/class/net/{iface}/wireless").is_dir()
def counters(iface: str) -> Optional[tuple]:
"""(rx_bytes, tx_bytes) straight off /sys, or None if the iface vanished."""
if not iface:
return None
try:
base = f"/sys/class/net/{iface}/statistics/"
with open(base + "rx_bytes") as f:
rx = int(f.read().strip())
with open(base + "tx_bytes") as f:
tx = int(f.read().strip())
except (OSError, ValueError):
return None
return rx, tx
def _num(text: str):
"""First number in a string, as int when it is whole.
`iw` is inconsistent across versions — this machine reports
`freq: 5180.0` where older builds print `freq: 5180`.
"""
m = re.search(r"-?\d+(?:\.\d+)?", text or "")
if not m:
return None
v = float(m.group(0))
return int(v) if v.is_integer() else v
def wifi_link(iface: str) -> dict:
"""SSID, signal, band and negotiated rates from `iw dev <iface> link`."""
raw = _run(["iw", "dev", iface, "link"])
if not raw or "Not connected" in raw:
return {}
info = {}
bssid = re.search(r"Connected to ([0-9a-fA-F:]{17})", raw)
if bssid:
info["bssid"] = bssid.group(1)
for line in raw.splitlines():
line = line.strip()
if line.startswith("SSID:"):
info["ssid"] = line.split(":", 1)[1].strip()
elif line.startswith("freq:"):
info["freq_mhz"] = _num(line)
elif line.startswith("signal:"):
info["signal_dbm"] = _num(line)
elif line.startswith("rx bitrate:"):
info["rx_mbps"] = _num(line)
elif line.startswith("tx bitrate:"):
info["tx_mbps"] = _num(line)
width = re.search(r"(\d+)MHz", line)
if width:
info["width_mhz"] = int(width.group(1))
for std, tag in (("HE", "802.11ax"), ("VHT", "802.11ac"), ("HT", "802.11n")):
if f"{std}-MCS" in line:
info["standard"] = tag
break
freq = info.get("freq_mhz")
if freq:
info["band"] = "6 GHz" if freq >= 5955 else "5 GHz" if freq >= 4900 else "2.4 GHz"
info["channel"] = _freq_to_channel(freq)
return info
def _freq_to_channel(freq) -> Optional[int]:
f = int(freq)
if f == 2484:
return 14
if 2412 <= f <= 2472:
return (f - 2407) // 5
if 5160 <= f <= 5885:
return (f - 5000) // 5
if 5955 <= f <= 7115:
return (f - 5950) // 5
return None
def wifi_station(iface: str) -> dict:
"""Airtime health from `iw station dump`.
Retries and failures are why Wi-Fi feels slow while the signal bar still
looks full, so they are worth the extra call.
"""
raw = _run(["iw", "dev", iface, "station", "dump"])
if not raw:
return {}
fields = {
"tx retries:": "tx_retries",
"tx failed:": "tx_failed",
"beacon loss:": "beacon_loss",
"rx drop misc:": "rx_drop_misc",
"tx packets:": "tx_packets",
"rx packets:": "rx_packets",
"signal avg:": "signal_avg_dbm",
"inactive time:": "inactive_ms",
}
out = {}
for line in raw.splitlines():
s = line.strip()
for prefix, key in fields.items():
if s.startswith(prefix):
v = _num(s[len(prefix):])
if v is not None:
out[key] = v
break
return out
def connection_name(iface: str) -> str:
"""The name NetworkManager shows, which is what the user calls this network."""
raw = _run(["nmcli", "-t", "-f", "GENERAL.CONNECTION", "dev", "show", iface])
if not raw:
return ""
for line in raw.splitlines():
if line.startswith("GENERAL.CONNECTION:"):
name = line.split(":", 1)[1].strip()
return "" if name in ("", "--") else name
return ""
# The connection's name is a NetworkManager fact: it changes when the
# route does, or when the user renames it, and nmcli is a D-Bus client
# that costs ~27 ms per call on this laptop — three quarters of what a
# whole snapshot cost when it was asked twice a second. The name is
# cached per (interface, gateway, BSSID) and re-asked on that key
# changing or every NAME_CACHE_TTL_S, whichever comes first.
NAME_CACHE_TTL_S = 60.0
_name_cache = {} # iface -> (key, name, expires_at)
def connection_name_cached(iface: str, key, now: float = None,
ttl: float = NAME_CACHE_TTL_S) -> str:
now = time.time() if now is None else now
hit = _name_cache.get(iface)
if hit and hit[0] == key and now < hit[2]:
return hit[1]
name = connection_name(iface)
_name_cache[iface] = (key, name, now + ttl)
return name
def snapshot(anchor: str = "1.1.1.1") -> dict:
"""Everything about the local end, in one call, safe to run twice a second."""
route = route_to(anchor)
iface = route.get("iface", "")
snap = {
"iface": iface,
"gateway": route.get("gateway", ""),
"src": route.get("src", ""),
"kind": "none",
}
if not iface:
return snap
snap["kind"] = "wifi" if is_wireless(iface) else "ethernet"
if snap["kind"] == "wifi":
snap.update(wifi_link(iface))
snap["station"] = wifi_station(iface)
key = (iface, snap["gateway"], snap.get("bssid", ""))
snap["name"] = connection_name_cached(iface, key) or iface
return snap
# ------------------------------------------------------------- wan address
TRACE_URL = "https://speed.cloudflare.com/cdn-cgi/trace"
def parse_trace(text: str) -> Optional[dict]:
"""The address, country and edge from a cdn-cgi/trace response.
Every value is validated against its own shape and anything else the
response carries is discarded unread — an address `ipaddress` accepts,
a two-letter country, a short alphabetic colo code. Nothing that fails
its check is passed on, so no free text from the wire reaches the
shell. Input is bounded before it is split, so an oversized body costs
one slice.
The country and edge come free: they are already in the response the
reachability check fetches every hour. Reading them adds no request
and no new destination, which is the whole reason they are here rather
than from a geolocation service that would learn every user's address.
The edge is Cloudflare's, not the user's — it says which datacentre
answered, so present it as provenance and never as a location.
"""
out = {}
for line in text[:4096].splitlines()[:64]:
if line.startswith("ip="):
try:
addr = ipaddress.ip_address(line[3:].strip())
except ValueError:
return None
out["ip"] = str(addr)
out["family"] = "v6" if addr.version == 6 else "v4"
elif line.startswith("loc="):
# Cloudflare answers XX when it does not know, which is not a
# country and must not be shown as one.
code = line[4:].strip()
if len(code) == 2 and code.isascii() and code.isalpha() \
and code.isupper() and code != "XX":
out["country"] = code
elif line.startswith("colo="):
edge = line[5:].strip()
if 2 <= len(edge) <= 5 and edge.isascii() and edge.isalpha() \
and edge.isupper():
out["edge"] = edge
# The address is the point; country and edge are decoration on it.
return out if "ip" in out else None
def trace_verdict(raw) -> str:
"""Did the real internet answer? `open` | `intercepted` | `silent`.
The same fetch that reads the WAN address is also the only thing here
that can tell the real internet from something standing in for it. A
probe reply proves a packet came back; it does not prove what sent it.
A captive portal, a transparent proxy or any middlebox will happily
complete a handshake and answer for an address it does not own — which
is how an unauthenticated hotel network produced a healthy-looking
internet leg with no internet behind it.
`open` is the only positive claim, and it needs the response to parse as
a trace with an address `ipaddress` accepts. Something that answered with
anything else is `intercepted`. Nothing at all is `silent`.
Honest limit: the fetch runs `curl -f`, so a portal that answers with a
4xx/5xx, a redirect with an empty body, or a certificate that does not
validate for the host all come back as nothing — `silent`. In practice
`intercepted` needs a portal that serves a 200 over a certificate valid
for speed.cloudflare.com, which is rare. The captive decision does not
care (both verdicts count against `open`); only this label does.
"""
if not raw:
return "silent"
return "open" if parse_trace(raw) else "intercepted"
def reachability() -> dict:
"""One reachability check: the verdict, plus the address when proven.
This is also where the WAN address comes from — the `proof` — so the
machine's address is only ever asked of a host the daemon talks to
anyway, never of an ifconfig-style third party. The URL is our
constant, never anything a response handed us. Cadence is the caller's
(CaptiveWatch): on every new network, hourly once the internet has
answered, every 30 s only while it has not.
"""
raw = _run(["curl", "-sf", "--proto", "=https", "--max-time", "5",
"--max-filesize", "4096", TRACE_URL], timeout=8.0)
verdict = trace_verdict(raw)
return {"verdict": verdict,
"proof": parse_trace(raw) if verdict == "open" else None}
plugins/io.github.x3me.nexthop/nexthopd/paths.py
+91
View File
@@ -0,0 +1,91 @@
"""Where Nexthop keeps its state.
Two directories since 0.2.22. The state dir (XDG_STATE_HOME, like the rest
of Omarchy) holds what must outlive the session: history.db, the
config.json the panel writes for the daemon, and the lock so two daemons
never fight. The runtime dir (XDG_RUNTIME_DIR, a per-user tmpfs) holds the
three snapshots the daemon rewrites continuously — live.json twice a
second, apps.json every three, recent.json every five. They are derived,
session-scoped, and worth nothing after a reboot, and on this laptop's
btrfs each 2.5 KB rewrite cost 62.5 KiB at the block layer with fsync:
14.65 GB a day for a bar widget, measured from the kernel's own counter.
On tmpfs it is nothing. Without XDG_RUNTIME_DIR they fall back to the
state dir, so reader and writer always agree in one session environment.
"""
import os
from pathlib import Path
def state_dir() -> Path:
base = os.environ.get("XDG_STATE_HOME") or os.path.join(Path.home(), ".local", "state")
return Path(base) / "nexthop"
def ensure_state_dir() -> Path:
"""Create the state dir, private to the user (0700).
The files inside carry the daemon's pid and the version string the
shell service uses to authorize a SIGTERM — nothing another account
has any business reading, let alone writing.
"""
d = state_dir()
d.mkdir(parents=True, exist_ok=True)
try:
d.chmod(0o700)
except OSError:
pass
return d
def runtime_dir() -> Path:
"""The volatile snapshots' home: the session's tmpfs, else the state dir."""
base = os.environ.get("XDG_RUNTIME_DIR")
if base and os.path.isdir(base):
return Path(base) / "nexthop"
return state_dir()
def ensure_runtime_dir() -> Path:
"""Create the runtime dir, private to the user like the state dir."""
d = runtime_dir()
d.mkdir(parents=True, exist_ok=True, mode=0o700)
try:
d.chmod(0o700)
except OSError:
pass
return d
LIVE = "live.json"
RECENT = "recent.json"
APPS = "apps.json"
DB = "history.db"
LOCK = "nexthopd.lock"
def live_path() -> Path:
return runtime_dir() / LIVE
def recent_path() -> Path:
return runtime_dir() / RECENT
def apps_path() -> Path:
return runtime_dir() / APPS
def manifest_path() -> Path:
"""The plugin's own manifest, beside this package rather than in the
state dir — the shell service reads it to spot a fast-forwarded
checkout."""
return Path(__file__).resolve().parent.parent / "manifest.json"
def db_path() -> Path:
return state_dir() / DB
def lock_path() -> Path:
return state_dir() / LOCK
plugins/io.github.x3me.nexthop/nexthopd/probes.py
+484
View File
@@ -0,0 +1,484 @@
"""Persistent ping and TCP-handshake probes, and the rolling windows they feed.
One long-lived `ping` process per target rather than one process per sample.
At two samples a second, spawning a process each time would mean 172,800
forks a day inside a laptop's idle budget; `ping -i` already does the timing
for us, and `-O` makes it say so out loud when a packet goes missing.
"""
import collections
import re
import shutil
import socket
import statistics
import subprocess
import threading
import time
from collections import deque
def nearest_rank(ordered, p: float):
"""Nearest-rank percentile of a NON-EMPTY sorted sequence.
One implementation shared by the probe stats, the per-app socket stats
and the speed baseline, so the three cannot drift apart — they read the
same figure off the same rule. The index formula already collapses to
element 0 for a single-element input, so no length special-case.
"""
i = min(len(ordered) - 1, max(0, int(round(p * (len(ordered) - 1)))))
return ordered[i]
# [1787562260.703963] 64 bytes from 10.10.0.1: icmp_seq=1 ttl=64 time=9.13 ms
RE_REPLY = re.compile(r"^\[(\d+\.\d+)\].*icmp_seq=(\d+).*time=([\d.]+)\s*ms")
# [1787562369.690501] no answer yet for icmp_seq=1
RE_PENDING = re.compile(r"^\[(\d+\.\d+)\]\s+no answer yet for icmp_seq=(\d+)")
# [...] From 10.10.0.147 icmp_seq=1 Destination Host Unreachable
RE_UNREACH = re.compile(r"^\[(\d+\.\d+)\].*icmp_seq=(\d+).*(?:Unreachable|unreachable)")
# How much recent history judges an instrument, and the fewest samples that
# can carry a judgement at all.
#
# Shared, deliberately, by the two places that ask "what has this instrument
# been doing lately": the bench, which ranks instruments over a window, and
# TcpProbe, which needs its own recent p50 to tell a retransmit from a slow
# path. They live here rather than on the bench because `instruments` imports
# this module and not the other way round, and an alias in the direction the
# imports already run is the only one Python will take.
#
# The point of aliasing rather than repeating the number: two literals plus a
# test catches drift on the next test run, an alias makes the drift
# impossible. `test_the_window_matches_what_the_bench_ranks_on` is kept even
# though it now passes by construction — it catches someone replacing an
# alias with a literal, which is the drift it was written against.
RECENT_WINDOW_S = 300.0
RECENT_MIN_SAMPLES = 8
class Series:
"""A rolling window of (timestamp, rtt_ms or None) for one target.
None means the probe went out and nothing came back. Keeping losses in
the same series as the replies is what lets a single pass compute both
latency and loss over any sub-window.
"""
def __init__(self, window_s: float = 1830.0):
self.window_s = window_s
self._samples = deque()
self._lock = threading.Lock()
def add(self, t: float, rtt, loaded: bool = False):
"""Record one probe result, tagged with whether the link was busy.
The tag is what makes bufferbloat visible: the same connection can
answer in 15 ms while idle and 300 ms while a download runs, and a
score built only on the idle number calls that line excellent right
up until someone uses it.
"""
with self._lock:
self._samples.append((t, rtt, bool(loaded)))
cutoff = t - self.window_s
while self._samples and self._samples[0][0] < cutoff:
self._samples.popleft()
def since(self, seconds: float):
cutoff = time.time() - seconds
with self._lock:
return [s for s in self._samples if s[0] >= cutoff]
def all(self):
with self._lock:
return list(self._samples)
@staticmethod
def split_by_load(samples):
"""(idle, loaded) — probes taken while the link was quiet vs busy.
Samples are indexed rather than unpacked throughout, so a caller
holding older two-element samples still reads as idle instead of
raising.
"""
idle = [s for s in samples if not (len(s) > 2 and s[2])]
loaded = [s for s in samples if len(s) > 2 and s[2]]
return idle, loaded
@staticmethod
def stats(samples) -> dict:
"""Latency percentiles, jitter and loss over the samples given.
Jitter is mean absolute difference between consecutive replies
(RFC 3550's IPDV), not standard deviation: a connection that
alternates 10/40/10/40 ms feels far worse than one that drifts
smoothly across the same range, and only IPDV says so.
"""
total = len(samples)
if total == 0:
return {"count": 0, "loss": None, "p50": None, "p75": None,
"p95": None, "jitter": None, "last": None, "max": None}
rtts = [s[1] for s in samples if s[1] is not None]
lost = total - len(rtts)
loss = lost / total
if not rtts:
return {"count": total, "loss": loss, "p50": None, "p75": None,
"p95": None, "jitter": None, "last": None, "max": None}
ordered = sorted(rtts)
deltas = [abs(rtts[i] - rtts[i - 1]) for i in range(1, len(rtts))]
last = next((x[1] for x in reversed(samples) if x[1] is not None), None)
return {
"count": total,
"loss": loss,
"p50": round(statistics.median(ordered), 2),
"p75": round(nearest_rank(ordered, 0.75), 2),
"p95": round(nearest_rank(ordered, 0.95), 2),
"max": round(ordered[-1], 2),
"jitter": round(statistics.fmean(deltas), 2) if deltas else 0.0,
"last": round(last, 2) if last is not None else None,
}
class PingProbe(threading.Thread):
"""Runs one `ping` forever, restarting it if it dies, feeding a Series.
A probe never raises into the daemon: if `ping` is missing, the target
stops resolving, or the interface goes away, the thread backs off and
keeps trying while the series simply records losses.
"""
def __init__(self, target: str, series: Series, interval_ms: int = 500,
name: str = "", loaded_fn=None):
super().__init__(name=f"probe-{name or target}", daemon=True)
self.target = target
self.series = series
# Asked at the moment a sample lands, so each probe is tagged with
# the link state it actually experienced rather than whatever the
# link was doing when the window is later read.
self.loaded_fn = loaded_fn
self.interval = max(0.2, interval_ms / 1000.0)
self._stop = threading.Event()
self._proc = None
# seq -> timestamp first seen unanswered, drained by _expire()
self._pending = {}
# seq -> when it was charged as lost. A packet the grace period gave
# up on can still be reported afterwards — the gateway's Destination
# Host Unreachable for it arrives later than the grace, in the real
# recording by 0.35 s — and without this it would be charged twice.
# Held for one further grace period, which is as long as a late report
# can be believed to belong to that packet.
self._charged = {}
def _loaded(self) -> bool:
try:
return bool(self.loaded_fn()) if self.loaded_fn else False
except Exception:
return False # a probe never raises into the daemon
def stop(self):
self._stop.set()
proc = self._proc
if proc and proc.poll() is None:
try:
proc.terminate()
except OSError:
pass
def set_interval(self, seconds: float):
"""Change cadence in place — a benched instrument idles, a seated
one probes at full rate, without tearing the thread down. `ping`
takes its interval on the command line, so the running process is
retired and the run loop respawns it with the new one."""
seconds = max(0.2, float(seconds))
if abs(seconds - self.interval) < 1e-9:
return
self.interval = seconds
proc = self._proc
if proc and proc.poll() is None:
try:
proc.terminate()
except OSError:
pass
def _reset_tracking(self):
"""Forget both maps together.
`ping` numbers from 1 again on every respawn, so a seq remembered past
the process that produced it would suppress a real loss on the next
one — turning a guard against overcharging into an undercount, which
is the same defect facing the other way.
"""
self._pending.clear()
self._charged.clear()
def _expire(self, now: float):
"""A packet still unanswered after the grace period is a lost packet.
`ping -O` reports "no answer yet" as soon as it sends the next probe,
but a slow reply can still land, so a pending seq is only counted as
lost once it is too old to come back.
"""
grace = self.interval * 2.5 + 1.0
for seq, t in list(self._pending.items()):
if now - t > grace:
del self._pending[seq]
self._charged[seq] = now
self.series.add(t, None, self._loaded())
# Bounded by the same clock that fills it: a seq stops being
# remembered once no report about it could still arrive.
for seq, t in list(self._charged.items()):
if now - t > grace:
del self._charged[seq]
def run(self):
backoff = 1.0
while not self._stop.is_set():
if not shutil.which("ping") or not self.target:
time.sleep(min(backoff, 30.0))
backoff = min(backoff * 2, 30.0)
continue
try:
self._run_once()
backoff = 1.0
except Exception:
# Never let a parse or spawn failure take the daemon with it.
time.sleep(min(backoff, 30.0))
backoff = min(backoff * 2, 30.0)
def _run_once(self):
cmd = ["ping", "-n", "-O", "-D", "-i", f"{self.interval:g}",
"-W", "1", self.target]
self._reset_tracking()
self._proc = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
text=True, bufsize=1,
)
try:
for line in self._proc.stdout:
if self._stop.is_set():
break
self._consume(line)
# ping exited: whatever was outstanding never arrived.
for seq, t in self._pending.items():
self.series.add(t, None, self._loaded())
self._reset_tracking()
finally:
proc, self._proc = self._proc, None
if proc:
try:
proc.terminate()
proc.wait(timeout=2)
except subprocess.TimeoutExpired:
# Would not go quietly: do not leave it running.
try:
proc.kill()
proc.wait(timeout=2)
except (OSError, subprocess.TimeoutExpired):
pass
except OSError:
pass
def _consume(self, line: str):
m = RE_REPLY.match(line)
if m:
t, seq, rtt = float(m.group(1)), int(m.group(2)), float(m.group(3))
self._pending.pop(seq, None)
# A reply this late cannot un-lose the packet — the window it
# belonged to has already been read — and recording the RTT as
# well would put two samples on the wire's one packet.
if seq not in self._charged:
self.series.add(t, rtt, self._loaded())
self._expire(t)
return
m = RE_UNREACH.match(line)
if m:
t, seq = float(m.group(1)), int(m.group(2))
self._pending.pop(seq, None)
if seq not in self._charged:
self.series.add(t, None, self._loaded())
self._expire(t)
return
m = RE_PENDING.match(line)
if m:
t, seq = float(m.group(1)), int(m.group(2))
# `ping -O` repeats "no answer yet" for the same seq, so one that
# has already been charged must not be put back on the pending
# list to be charged a second time.
if seq not in self._charged:
self._pending.setdefault(seq, t)
self._expire(t)
class TcpProbe(threading.Thread):
"""Connect-time RTT to the anchor's TLS port, feeding a Series.
ICMP measures what routers choose to answer, and they answer it fast:
many devices handle it in hardware, in an ASIC or via XDP, while real
traffic waits in the user-space path behind the queues that actually
hold it up. Anything on the way can also reply on the destination's
behalf, because there is nothing in ICMP to prove otherwise.
A TCP handshake cannot be shortcut that way. The SYN has to reach a
listener that completes it, over port 443 where the user's own traffic
goes, so its round trip is the one applications experience. One
connection per sample, opened and closed — no payload, no TLS, nothing
kept.
Since 0.2.0 these are seated instruments in the bench (instruments.py):
the two best of four feed the scored internet leg, so a TCP series moves
the score whenever it holds a seat. The anchor's ICMP figure is still
recorded beside it per minute (`lag_icmp`) so the switch stays auditable.
"""
CONNECT_TIMEOUT_S = 2.0
# Linux and macOS both start TCP's retransmit timer at one second, so a
# handshake that comes back at or past this did not measure a slow path:
# its SYN was dropped and the kernel sent another. The number is the
# kernel's constant, not the network's round trip, and folding it into a
# latency percentile reports the line as slow when what happened is that
# a packet was lost.
#
# The connect timeout was already drawing this line, in the wrong place
# and for the wrong reason: a handshake needing TWO retransmits waits
# 1 s + 2 s, exceeds CONNECT_TIMEOUT_S and is recorded as loss, while one
# needing a single retransmit returns at ~1 s and was recorded as a round
# trip. The same event, accounted two opposite ways, with the boundary
# wherever the timeout happened to fall.
# One initial RTO, with slop for timer granularity and scheduling. The
# first retransmit fires at 1000 ms on Linux, macOS and Windows alike.
RETRANSMIT_MARGIN_MS = 900.0
# The baseline is this instrument's own recent p50 over the shared window
# above — aliases, not copies, so "recent" cannot come to mean two things.
RETRANSMIT_WINDOW_S = RECENT_WINDOW_S
RETRANSMIT_MIN_SAMPLES = RECENT_MIN_SAMPLES
def __init__(self, target: str, series: Series, interval_s: float = 1.0,
name: str = "", loaded_fn=None, port: int = 443):
super().__init__(name=f"tcp-{name or target}", daemon=True)
self.target = target
self.port = port
self.series = series
self.interval = max(0.25, interval_s)
self.loaded_fn = loaded_fn
self._stop = threading.Event()
self.ever_connected = False
# The recent round trips this probe has actually seen, for the
# comparison above. Bounded, and its own — the series it feeds is
# merged with other instruments and cannot answer "what does THIS
# path usually do". Held as (when, rtt) so the window is a duration
# rather than a count, which is what makes it survive a cadence
# change: a benched instrument probes at a fraction of the rate.
self._recent = collections.deque(maxlen=1024)
# Samples this probe declined to call latency, so the reclassification
# can be seen rather than inferred from a loss rate. A dropped SYN and
# a slow line are different faults with different owners.
self.retransmits = 0
self.unclassified = 0
def stop(self):
self._stop.set()
def set_interval(self, seconds: float):
"""Picked up on the next cycle; nothing to tear down here."""
self.interval = max(0.25, float(seconds))
def _loaded(self) -> bool:
try:
return bool(self.loaded_fn()) if self.loaded_fn else False
except Exception:
return False
def _once(self):
started = time.time()
t0 = time.monotonic()
try:
sock = socket.create_connection((self.target, self.port),
timeout=self.CONNECT_TIMEOUT_S)
except (OSError, ValueError):
self.series.add(started, None, self._loaded())
return
rtt = (time.monotonic() - t0) * 1000.0
try:
sock.close()
except OSError:
pass
# The handshake completed, so the target is reachable, whatever the
# kernel had to do to get there.
self.ever_connected = True
verdict = self._classify(started, rtt)
if verdict == "retransmit":
# Loss on new connections, which is what it is. Recorded the same
# way a refused or timed-out connect already is, so it charges the
# loss term and Reliability rather than the latency percentiles.
self.retransmits += 1
self.series.add(started, None, self._loaded())
return
if verdict == "unknown":
# Past the floor before this probe has a baseline to judge it
# against. It is either a retransmit or a genuinely slow path and
# nothing here can tell which, so it is not recorded as either —
# inventing a loss and publishing a suspect latency are both
# claims, and the honest move is to make neither.
#
# It still feeds the baseline, and that is not an oversight. A
# link whose real round trip is past the floor — p50 1200 ms, say
# — has every sample land here, so a deque that only accepted
# classified samples would never reach its minimum, the baseline
# would never form, and the instrument would stay unclassified
# for ever: nothing recorded, count never growing, `penalty()`
# returning None, and the bench able neither to seat it nor to
# call it dead. A silent unrankable instrument, invisible because
# it is not failing, merely absent.
self.unclassified += 1
self._recent.append((started, rtt))
return
self._recent.append((started, rtt))
self.series.add(started, round(rtt, 2), self._loaded())
def _baseline_ms(self, now: float):
"""This instrument's own recent p50, or None while it has too few."""
cutoff = now - self.RETRANSMIT_WINDOW_S
recent = [rtt for t, rtt in self._recent if t >= cutoff]
if len(recent) < self.RETRANSMIT_MIN_SAMPLES:
return None
return statistics.median(recent)
def _classify(self, now: float, rtt_ms: float) -> str:
""""reply", "retransmit" or "unknown" for a handshake that completed.
A connect rescued by a retransmitted SYN is a lost packet, not a slow
path, and the threshold has to be relative or it mislabels distance as
loss: one full RTO ABOVE what this instrument usually sees. A satellite
link whose p50 is 600 ms gets a threshold of 1500, so a 1045 ms sample
there stays the measurement it is.
"""
if rtt_ms < self.RETRANSMIT_MARGIN_MS:
return "reply"
baseline = self._baseline_ms(now)
if baseline is None:
return "unknown"
# Note what does NOT reach the baseline once one exists: a sample this
# returns "retransmit" for. Feeding those back would raise the
# threshold on the instrument's own retransmits and the rule would
# quietly stop firing exactly where it is needed most.
return "retransmit" if rtt_ms >= baseline + self.RETRANSMIT_MARGIN_MS \
else "reply"
def run(self):
while not self._stop.is_set():
if not self.target:
self._stop.wait(5.0)
continue
t0 = time.monotonic()
try:
self._once()
except Exception:
# Never let a socket or DNS failure take the daemon with it.
pass
self._stop.wait(max(0.0, self.interval - (time.monotonic() - t0)))
plugins/io.github.x3me.nexthop/nexthopd/score.py
+553
View File
@@ -0,0 +1,553 @@
"""Turning measurements into the three component scores and the index.
The shape follows Orb: Responsiveness, Reliability and Speed each score
0-100, and the index is the weakest of them with a nudge from the other two
(`index`, 0.1.10 — a plain mean let one dead dimension hide behind two good
ones). Ranking a connection by its download number alone is exactly the
habit that leaves people with a fast line that feels broken on a video call.
Every threshold below is an anchor table rather than a formula. Anchors are
arguable in public, which is the point: someone who disagrees that 60 ms of
lag is a 78 can say so about one number instead of reverse-engineering a
curve.
"""
# Lag in ms -> Responsiveness score. Interpolated linearly between anchors.
LAG_ANCHORS = [
(10, 100), (20, 95), (35, 88), (60, 78), (100, 68),
(200, 50), (400, 30), (800, 10), (1500, 0),
]
# Fraction of the plan achieved -> Speed score, used only when the user has
# configured a plan. Deliberately forgiving in the middle: an ISP delivering
# 80% of a sold plan is doing fine, and a score that punished that would cry
# wolf every evening.
SPEED_ANCHORS = [
(0.0, 0), (0.1, 15), (0.25, 35), (0.5, 60),
(0.7, 75), (0.85, 88), (1.0, 96), (1.15, 100),
]
# The default basis: Mbps -> score, anchored to what applications need
# rather than to any plan. Speed has steep diminishing returns — 25 Mbps
# carries a 4K stream, ~100 feels instant for nearly everything, and past
# ~300 a person cannot tell the difference — so the curve saturates.
SPEED_ABS_DOWN = [
(0, 0), (5, 25), (25, 55), (50, 70), (100, 82),
(200, 90), (300, 94), (500, 98), (750, 100),
]
SPEED_ABS_UP = [
(0, 0), (2, 30), (5, 55), (10, 70), (20, 82),
(50, 92), (100, 100),
]
BANDS = [(90, "excellent"), (80, "good"), (70, "okay"), (50, "fair"), (0, "poor")]
def _interp(anchors, x):
if x <= anchors[0][0]:
return float(anchors[0][1])
if x >= anchors[-1][0]:
return float(anchors[-1][1])
for i in range(1, len(anchors)):
x0, y0 = anchors[i - 1]
x1, y1 = anchors[i]
if x <= x1:
span = x1 - x0
return float(y0 + (y1 - y0) * ((x - x0) / span if span else 0))
return float(anchors[-1][1])
# What one lost packet costs, in milliseconds. It is a retransmit timeout,
# so it scales with the link's own round trip rather than being a constant:
# RTO is roughly SRTT + 4·RTTVAR, and nothing recovers faster than Linux's
# 200 ms floor. The stall factor is on top, because a drop costs more than
# the resent packet — everything behind it waits (head-of-line blocking)
# and the congestion window has to climb back.
LOSS_RTO_FLOOR_MS = 200.0
LOSS_RTT_MULTIPLIER = 3.0
LOSS_STALL_FACTOR = 5.0
def loss_cost_ms(p75_ms: float) -> float:
"""Milliseconds of felt lag per unit of loss, for a link this fast.
Below ~67 ms p75 this returns 1000, which is exactly the flat constant
it replaces — so ordinary connections score as they always did. Above
it the charge grows with the round trip, which is the part the constant
got wrong: on a 600 ms satellite link a dropped packet does not cost the
same 10 ms per percent that it costs on fibre.
"""
return LOSS_STALL_FACTOR * max(LOSS_RTO_FLOOR_MS,
LOSS_RTT_MULTIPLIER * max(0.0, p75_ms))
def lag_ms(stats: dict):
"""One number for how the connection feels, in milliseconds.
Latency alone under-reports: a link that is 10 ms most of the time but
swings to 90 ms and drops a packet every few seconds feels much worse
than its median suggests. So lag leans on p75 rather than the median,
adds the jitter the user actually perceives, and charges for loss at a
rate that reflects a retransmit round trip on THIS link.
"""
if not stats or stats.get("count", 0) == 0:
return None
loss = stats.get("loss") or 0.0
if stats.get("p75") is None:
# Everything was lost. There is no latency to report, only a verdict.
return None if loss < 1.0 else 1500.0
base = stats["p75"]
jitter = stats.get("jitter") or 0.0
return round(base + 1.5 * jitter + loss * loss_cost_ms(base), 1)
def lag_band(stats: dict) -> dict:
"""Lag at three latency percentiles: best, typical, worst.
All three go through the same fold, differing only in which percentile
they lean on, and that is the whole point. The panel used to pair a
loss-charged "typical" with raw millisecond figures either side of it,
so a lossy link displayed "best 4 · typical 644 · worst 26" — three
numbers that cannot all be true at once, because two were round trips
and one was a composite.
Sharing the fold makes the ordering hold by construction and makes loss
move all three together, which is what a reader assumes a range means.
"""
if not stats or stats.get("count", 0) == 0:
return {"best": None, "typical": None, "worst": None}
if stats.get("p75") is None:
# Everything in the window was lost. `lag_ms` answers 1500 here so
# Responsiveness lands on zero, which is its job — but 1500 is an
# anchor, not a measurement, and the panel used to print it three
# times as though the link were replying slowly. There is no latency
# to display, so display none.
return {"best": None, "typical": None, "worst": None}
out = {}
prev = None
for name, key in (("best", "p50"), ("typical", "p75"), ("worst", "p95")):
v = lag_ms(dict(stats, p75=stats.get(key)))
# p95 can equal p75 on a short window, and a percentile can be
# missing; neither may let the range read backwards.
if v is not None and prev is not None:
v = max(v, prev)
out[name] = v
if v is not None:
prev = v
return out
def responsiveness(lag):
if lag is None:
return 0.0
return round(_interp(LAG_ANCHORS, lag), 1)
RELIABILITY_WINDOW_S = 24 * 3600
# A self-healed interruption is real but not as bad as being down, so its
# time is charged at a discount.
DISRUPTION_TIME_WEIGHT = 0.5
# Each interruption also costs recovery beyond its own length — a dropped
# call is redialled, a stream rebuffers, a download restarts — so every
# event carries this much equivalent disruption. Expressed in SECONDS on
# purpose: a penalty in raw points cannot be compared with downtime, which
# is exactly how the old flat "6 points per disruption" ended up charging a
# brief blip more than an hour offline.
DISRUPTION_RECOVERY_S = 300.0
# Backstop so a pathological count can never dominate the component.
DISRUPTION_MAX_PENALTY = 25.0
def reliability(outage_fraction: float, disruptions: int, covered: bool = True,
disruption_fraction: float = 0.0,
window_s: float = RELIABILITY_WINDOW_S):
"""Uptime, not smoothness — everything charged in one currency: time.
Orb moved reliability to bite only during true outages, and that is the
right call: a wobbly ten minutes is already punished by responsiveness,
and double-counting it made the overall score swing on a single bad
evening. Here an outage is total loss on the wan leg; a disruption is a
shorter interruption that resolved on its own.
Both are now charged by DURATION. They used to be charged in different
currencies — outages by their share of the window, disruptions at a flat
6 points each — and the units did not meet: over a 24 h window one
ten-minute outage cost 0.7 points while three self-healed blips cost 18,
so the milder event was punished twenty-six times harder, and seventeen
blips zeroed the component outright. Time is the honest unit for "how
much of today was this connection unusable", and an event's recovery
cost is expressed in seconds so it lands on the same scale.
"""
if not covered:
return 100.0
window = window_s if window_s and window_s > 0 else RELIABILITY_WINDOW_S
down = max(0.0, min(1.0, outage_fraction))
disrupted_s = (max(0.0, min(1.0, disruption_fraction)) * window
+ max(0, disruptions) * DISRUPTION_RECOVERY_S)
penalty = min(DISRUPTION_MAX_PENALTY,
100.0 * DISRUPTION_TIME_WEIGHT * min(1.0, disrupted_s / window))
score = 100.0 - 100.0 * down - penalty
return round(max(0.0, min(100.0, score)), 1)
def speed_absolute(down_mbps, up_mbps):
"""Is it fast enough — scored against what applications need.
Download is weighted 3:1 over upload. That is not a claim that upload
matters less in general — it is that most lines are asymmetric by
design, so equal weighting would score every ordinary connection as
broken.
"""
if down_mbps is None:
return None
parts = [(_interp(SPEED_ABS_DOWN, down_mbps), 3.0)]
if up_mbps is not None:
parts.append((_interp(SPEED_ABS_UP, up_mbps), 1.0))
weighted = sum(v * w for v, w in parts)
return round(weighted / sum(w for _, w in parts), 1)
# A Speed figure may be reported and still be too thin to set the headline.
# Two samples is the floor because the guard above it is a median: with one
# sample there is no median to take, so the single check IS the verdict —
# and the least trustworthy check of all is the first one after joining a
# network, taken while the link is still settling.
MIN_SPEED_SAMPLES = 2
# A saturating test that read at least this much more than the everyday
# basis has disproved it. Peak stays unscored — a manual test must not
# flatter the score — but it can withdraw a figure it contradicts.
PEAK_CONTRADICTION_RATIO = 2.0
def speed_scored(down_mbps, samples: int, peak_down=None) -> bool:
"""May this Speed figure set the index, or only be displayed?
The index is weakest-link, so whichever component is lowest becomes the
headline. Responsiveness and Reliability are built from thousands of
probes a minute; Speed is one 12 MB sample an hour. Letting the thinnest
input hold a veto is how a healthy 380 Mbps line reported POOR off a
single check taken 55 seconds after associating, while a peak test on
the same line minutes later read four times higher.
So the rule is not new weighting, it is eligibility: a figure that is
under-sampled, or contradicted by a faster measurement of the same line,
is shown with its reason and left out of the index. `index` already
skips a component it does not have rather than inventing one — this
gives it the same honesty for a component we have but do not trust.
"""
if down_mbps is None:
return False
if samples < MIN_SPEED_SAMPLES:
return False
if peak_down and peak_down >= down_mbps * PEAK_CONTRADICTION_RATIO:
return False
return True
def degradation_penalty(down_mbps, baseline_down):
"""Is it normal for this network — a penalty for big drops only.
The baseline is the connection's own recent p90. Sharing an office line
means honest hour-to-hour variance, so nothing below a 40% shortfall
counts; from there the penalty grows to 35 points at zero. This is what
catches "we normally get 300 here and today it is 60" on a line whose
absolute score would still look comfortable.
"""
if down_mbps is None or not baseline_down or baseline_down <= 0:
return 0.0
ratio = down_mbps / baseline_down
if ratio >= 0.6:
return 0.0
return round((0.6 - ratio) / 0.6 * 35.0, 1)
def speed(down_mbps, up_mbps, plan_down=0, plan_up=0, baseline_down=None):
"""The Speed component.
With a configured plan: scored against the plan (ISP accountability —
opt-in, because almost nobody configures a plan and shared office lines
have no meaningful one). Without: the absolute experience curve, minus
the degradation penalty against the connection's own baseline.
"""
if down_mbps is None:
return None
if plan_down and plan_down > 0:
ratios = [(down_mbps / plan_down, 3.0)]
if plan_up and plan_up > 0 and up_mbps is not None:
ratios.append((up_mbps / plan_up, 1.0))
weighted = sum(_interp(SPEED_ANCHORS, r) * w for r, w in ratios)
return round(weighted / sum(w for _, w in ratios), 1)
base = speed_absolute(down_mbps, up_mbps)
if base is None:
return None
return round(max(0.0, base - degradation_penalty(down_mbps, baseline_down)), 1)
# How much of the index the worst component owns. The remainder lets the
# other two nudge it up a little, so "everything else is excellent" still
# reads differently from "everything is mediocre".
INDEX_WORST_WEIGHT = 0.92
def index(resp, rel, spd):
"""Weakest-link, skipping any component we genuinely cannot measure.
A mean let one broken dimension hide behind two good ones: a line
scoring Responsiveness 40, Reliability 100, Speed 95 averaged to 78 and
read as "okay" — while video calls on it did not work. That is exactly
the habit this module exists to avoid, reintroduced at the last step.
People experience the bottleneck, not the average, so the worst
component sets the number and the others only nudge it.
This also puts us where the rest of the field is: Pulse aggregates
weakest-link (validated against a real fleet) and IETF
draft-ietf-ippm-qoo takes a strict minimum. A mean was the outlier.
Scoring an unmeasured component as zero would be a lie; scoring it as
100 would be a different lie. Leaving it out and saying so is honest,
and it means the index is useful within seconds of starting rather than
after the first speed test lands.
"""
parts = [p for p in (resp, rel, spd) if p is not None]
if not parts:
return None
worst = min(parts)
others = list(parts)
others.remove(worst) # by equality: one instance, ties keep the rest
if not others:
return int(round(worst))
rest = sum(others) / len(others)
return int(round(INDEX_WORST_WEIGHT * worst + (1.0 - INDEX_WORST_WEIGHT) * rest))
# The states in which a headline index is not a current reading.
OUTAGE_STATES = ("local-down", "wan-down")
def scored_now(state):
"""Whether an index may stand as the headline in this state.
Not while a leg is confirmed down. Every input to the index describes a
window that mostly predates the outage: Lag reads 30 s that still holds
pre-outage replies, and Reliability charges the downtime against 24 h,
where a minute is 0.07 % and rounds away. Speed is skipped honestly.
Weakest-link over two components that both still read 100 therefore
reports 100 — seen in the wild on a real 61 s Wi-Fi drop, the panel
showing EXPERIENCE 100 directly beneath its own ROUTER UNREACHABLE.
Withheld rather than lowered, because any number chosen here would be
invented, and the state is already the honest headline: the panel draws
the verdict beside it and the bar counts the outage. Same rule as
scoring None rather than fabricating a figure.
A quiet spell is deliberately not an outage. gateway-quiet and
icmp-quiet leave the state calm precisely because traffic is still
crossing the leg, so the index keeps standing there and should.
"""
return state not in OUTAGE_STATES
def band(score):
if score is None:
return "unknown"
for floor, name in BANDS:
if score >= floor:
return name
return "poor"
# Below this the two legs are indistinguishable at our resolution, and a
# near-zero ISP leg is physically possible (an anchor a hop past the
# gateway). Above it, the router answering slower than the internet behind
# it means the two independent distributions disagree and the subtraction is
# void — not that the ISP adds nothing.
WAN_INVERSION_TOLERANCE_MS = 1.0
def wan_point_ms(total_ms, local_ms):
"""The ISP leg for ONE pair of readings, or None when it says nothing.
The same rule `wan_from` applies per statistic, factored out so the
per-point series in recent.json and the per-window statistics cannot
drift apart. A gateway that answers slower than the internet behind it
is common — plenty of them deprioritise ICMP addressed to themselves —
and the subtraction has nothing to say about the line when it happens,
so the answer is None rather than a clamped zero.
"""
if total_ms is None or local_ms is None:
return None
if local_ms > total_ms + WAN_INVERSION_TOLERANCE_MS:
return None
return round(max(0.0, total_ms - local_ms), 2)
def wan_from(total: dict, local: dict) -> dict:
"""The ISP leg: what is left of the round trip once the router's share is gone.
The two probes are not synchronised, so this subtracts distributions
rather than individual packets — p50 from p50, p75 from p75. Loss on the
wan leg is whatever the internet probe lost beyond what the router probe
lost, since loss on the local link shows up in both.
"""
out = {"count": total.get("count", 0)}
prev = 0.0
for key in ("p50", "p75", "p95", "max"):
t, l = total.get(key), local.get(key)
# One refusal, in one place — `wan_point_ms` — for both reasons it
# refuses, because a rule whose whole job is to withhold must not
# have a second copy that can forget to.
#
# It withholds when the router answered SLOWER than the internet
# behind it, because `total = local + wan` does not hold and the
# subtraction has nothing to say. Clamping the negative to zero used
# to report the ISP leg as 0.0 ms — the best possible answer, from an
# invalid measurement, on the number the whole panel is built around.
# It happens for a real reason: plenty of gateways deprioritise or
# rate-limit ICMP addressed to themselves, so their own replies are
# slow while everything they forward is fast. That says something
# about the gateway's control plane, not about the link.
#
# And when either reading is missing: unknown, not zero. Substituting
# 0 for a local statistic we do not have made the derived leg equal
# the whole round trip, so a silent gateway produced a confident,
# healthy-looking internet figure that was really the total wearing
# the wan leg's label.
v = wan_point_ms(t, l)
if v is None:
out[key] = None
continue
# Subtracting two independent distributions statistic-by-statistic
# can invert the order (a wan p95 below the wan p50) when the local
# leg's tail is fatter than the total's. Each statistic is floored
# at the one before it so the derived leg reads like a distribution.
# The floor has to carry forward FLOORED, not raw: `prev` is what the
# previous statistic ended up reporting, so a p95 that subtracts lower
# than the p50 still reads as a distribution.
v = max(prev, v)
out[key] = round(v, 2)
prev = v
t, l = total.get("last"), local.get("last")
if t is None or (l is not None and l > t + WAN_INVERSION_TOLERANCE_MS):
out["last"] = None
else:
out["last"] = round(max(0.0, t - (l or 0.0)), 2)
# Jitter does not subtract: variance on the local link propagates into
# the total, so the honest reading is "no less than the total's jitter
# minus the local's", floored at zero.
tj, lj = total.get("jitter"), local.get("jitter")
out["jitter"] = None if tj is None else round(max(0.0, tj - (lj or 0.0)), 2)
tl, ll = total.get("loss"), local.get("loss")
out["loss"] = None if tl is None else max(0.0, tl - (ll or 0.0))
return out
# How close to the idle baseline counts as drained. A queue does not empty
# to the exact millisecond it started from, and demanding that would report
# "never recovered" on a link that plainly had.
DRAIN_TOLERANCE = 1.25
# Longest drain worth reporting. Past this the link did not recover from a
# burst, it is simply in a different state, and calling that a drain time
# would flatter it.
DRAIN_MAX_S = 30.0
def drain_after_load(samples, baseline_ms: float) -> dict:
"""How long latency took to fall back to baseline after load stopped.
Bufferbloat is reported everywhere as a depth — how much delay a busy
link adds. Depth alone cannot tell apart two links a user experiences
very differently: one whose queue fills and empties the instant traffic
stops, and one that stays full for seconds afterwards. The second ruins
a call after the download has finished; the first does not.
Pure, and fed the sample stream it already has: `(t, rtt, loaded)`
tuples, where the third element is the load tag added in 0.1.11. No
extra traffic, and nothing to schedule — the user's own usage supplies
the burst.
Returns `{"ms": None}` when there is nothing to say, which is most of
the time: no burst in the window, or the link never came back inside
`DRAIN_MAX_S`, or the baseline is unknown.
"""
out = {"ms": None, "settled": None}
if not samples or not baseline_ms or baseline_ms <= 0:
return out
# The most recent load -> idle transition, which is the only one whose
# recovery is still visible in this window.
last_loaded = None
for i, sm in enumerate(samples):
if len(sm) > 2 and sm[2]:
last_loaded = i
if last_loaded is None or last_loaded == len(samples) - 1:
return out # no burst, or still under load
ended_t = samples[last_loaded][0]
target = baseline_ms * DRAIN_TOLERANCE
for sm in samples[last_loaded + 1:]:
if sm[1] is None:
continue # a lost probe says nothing either way
if sm[1] <= target:
span = sm[0] - ended_t
if span > DRAIN_MAX_S:
return out
out["ms"] = round(max(0.0, span) * 1000.0, 0)
out["settled"] = True
return out
# Still above the baseline at the end of the window: report the floor it
# has already exceeded rather than a number implying it recovered.
span = samples[-1][0] - ended_t
if 0 < span <= DRAIN_MAX_S:
out["ms"] = round(span * 1000.0, 0)
out["settled"] = False
return out
# Queueing delay, in milliseconds, that separates a link carrying traffic
# comfortably from one holding packets up. Deliberately the same shape as
# the bufferbloat grades and the Latency tab's copy, and deliberately about
# the ABSOLUTE delay rather than its share of the round trip: a socket to
# another continent is mostly distance, and a proportion would call that
# congested.
PRESSURE_BUSY_MS = 10.0
PRESSURE_CONGESTED_MS = 30.0
def pressure(socket_queue_ms=None, loaded_ms=None, idle_ms=None) -> dict:
"""What the connection is doing RIGHT NOW, as opposed to lately.
The index cannot answer this and is not meant to. It is a weakest-link
score over three components, one of which — Speed — moves at
content-check cadence, so when it is the weakest the index barely
responds to anything else. Measured live: a saturating test drove
Responsiveness down 14.6 points while the index moved from 75.0 to
75.0, because Speed sat permanently lowest at 73.1. Both numbers were
correct; neither answered "is it bad right now".
So this is a separate, fast channel rather than a change to the index.
It reports queueing delay, which is the thing a user actually feels
during a burst, and it prefers the figure taken from their own TCP
connections (`sockets.queue_p50`, the kernel's own timing) over our
probes' loaded-minus-idle difference, because real traffic to real
destinations beats an inference from two sample populations.
Returns `state: None` when neither source can say, which is honest and
common on an idle machine with nothing to measure.
"""
src, q = None, None
if socket_queue_ms is not None and socket_queue_ms >= 0:
src, q = "sockets", float(socket_queue_ms)
elif (loaded_ms is not None and idle_ms is not None
and loaded_ms >= idle_ms):
# Only when the difference points the right way; queueing cannot be
# negative, and a negative difference means the split is unreliable
# rather than that load helped.
src, q = "probes", float(loaded_ms - idle_ms)
if q is None:
return {"state": None, "queue_ms": None, "source": None}
if q >= PRESSURE_CONGESTED_MS:
state = "congested"
elif q >= PRESSURE_BUSY_MS:
state = "busy"
else:
state = "clear"
return {"state": state, "queue_ms": round(q, 1), "source": src}
plugins/io.github.x3me.nexthop/nexthopd/speedtest.py
+561
View File
@@ -0,0 +1,561 @@
"""Speed measurement, two kinds.
Content speed: a short ranged download and a small upload, on a schedule, small enough to
be honest about the connection without being a burden on it. This is what
feeds the Speed score, following Orb's split — score the everyday number,
keep the fireworks manual.
Peak speed: saturates the line, only ever on demand. Prefers the official
Ookla CLI when installed (server choice, shareable result), falls back to
Cloudflare's endpoints via curl, then fast.com via the same API Omarchy's
built-in speed test uses. Both fallbacks need nothing installed beyond curl.
Loaded latency is sampled during the peak download by the daemon's existing
probes, not here — the test just records the window it ran in.
"""
import ipaddress
import json
import shutil
import socket
import subprocess
import threading
import time
from typing import Optional
from urllib.parse import urlparse
CLOUDFLARE_DOWN = "https://speed.cloudflare.com/__down?bytes={n}"
CLOUDFLARE_UP = "https://speed.cloudflare.com/__up"
# The token fast.com's own web client uses; Omarchy's built-in speed test
# ships the same one.
FAST_API = ("https://api.fast.com/netflix/speedtest/v2"
"?https=true&token=YXNkZmFzZGxmbnNkYWZoYXNkZmhrYWxm&urlCount=3")
def vet_target(url: str):
"""(url, --resolve argument) for a target we will fetch, else None.
Only for URLs WE DID NOT CHOOSE. fast.com nominates its own download
hosts, so that JSON decides what this daemon connects to, and it has
to be treated as hostile input rather than as a list of Netflix
servers. Three things must hold:
1. the scheme is https, so a nominated target cannot downgrade the
transfer to plaintext or hand curl a `file://` path;
2. EVERY address the host resolves to is public, so a speed test can
never be aimed at a router's admin page, a service on loopback,
or a link-local metadata address;
3. the address that passed (2) is the one curl actually connects to.
The third is the point most of this class gets wrong: resolving here
and letting curl resolve again is a check-then-use race, and a DNS
answer that returns a public address to us and a private one to curl
wins it. Pinning the vetted addresses with --resolve closes that
window, the same way every other read in this daemon is enforced on
the thing actually used rather than on a name looked up earlier.
"""
try:
parsed = urlparse(url)
except ValueError:
return None
if parsed.scheme != "https" or not parsed.hostname:
return None
port = parsed.port or 443
try:
infos = socket.getaddrinfo(parsed.hostname, port, type=socket.SOCK_STREAM)
except (OSError, ValueError, UnicodeError):
return None
addrs = []
for info in infos:
try:
ip = ipaddress.ip_address(info[4][0])
except ValueError:
return None
if ip.version == 6 and ip.ipv4_mapped is not None:
ip = ip.ipv4_mapped
# Spelled out rather than leaning on is_global alone, whose range
# table has been corrected across Python versions we may run on.
if (ip.is_private or ip.is_loopback or ip.is_link_local
or ip.is_multicast or ip.is_reserved or ip.is_unspecified):
return None
addrs.append(str(ip))
if not addrs:
return None
return url, "%s:%d:%s" % (parsed.hostname, port, ",".join(addrs))
def _curl(args, timeout) -> Optional[subprocess.CompletedProcess]:
if not shutil.which("curl"):
return None
try:
# --proto =https refuses anything but TLS even if a target or a
# server tries something else; we never pass -L, so there is no
# redirect for it to follow either.
return subprocess.run(["curl", "-fsS", "--proto", "=https",
"--max-time", str(int(timeout))] + args,
capture_output=True, text=True, timeout=timeout + 5,
check=False)
except (subprocess.TimeoutExpired, OSError):
return None
# A transfer shorter than this carried too few bytes for its own duration to
# be worth dividing by: the timing error, not the line, would set the answer.
MIN_TIMED_WINDOW_S = 0.05
def _rate_over_payload(size: float, t_total: float, t_payload_start: float):
"""Mbps over the part of the request that actually carried bytes.
curl's own `speed_download` divides the bytes by the WHOLE request —
DNS, the TCP connect, the TLS handshake and the wait for the first byte
included. None of that carried payload, and none of it shrinks when the
line gets faster, so it is a fixed tax on a measurement whose useful part
keeps getting shorter: a 3 MB stream is ~240 ms of payload on a 400 Mbps
line and ~107 ms on a gigabit one, against the same ~70-90 ms of setup.
That is not noise. It is a bias that grows with the quantity being
measured, which made the check read a 900 Mbps line as roughly 220 and
put a ceiling near 480 on a scale whose top two anchors are 500 and 750.
Dividing by a window instead of by the total is only honest while the
window is long enough to divide by, so a sample too short to time is
withheld rather than published — the same rule the rest of the daemon
uses for a figure it cannot stand behind.
"""
window = t_total - t_payload_start
if window < MIN_TIMED_WINDOW_S or size <= 0:
return None
return size * 8 / 1e6 / window
def _curl_timed_download(url: str, timeout: float, resolve: str = None):
"""(mbps, bytes), timed over the payload rather than the whole request."""
pin = ["--resolve", resolve] if resolve else []
r = _curl(pin + ["-o", "/dev/null",
"-w", "%{size_download} %{time_total} %{time_starttransfer}",
url],
timeout)
if not r or r.returncode != 0:
return None, 0
try:
size, t_total, t_start = (float(x) for x in r.stdout.split())
except ValueError:
return None, 0
return _rate_over_payload(size, t_total, t_start), int(size)
def _upload_argv(url: str, timeout: float):
"""The one upload invocation, shared by the single and parallel forms.
The body is piped in — pointing curl at /dev/zero directly would have it
read the file to its end, which /dev/zero does not have."""
return ["curl", "-fsS", "--proto", "=https", "--max-time", str(int(timeout)),
"-o", "/dev/null", "-X", "POST", "--data-binary", "@-",
"-H", "Content-Type: application/octet-stream",
# Not time_starttransfer: on a POST that is the first byte of the
# RESPONSE, and against speed.cloudflare.com it arrives right after
# the handshake (the 100-continue), not after the body. The TLS
# handshake completing is when this request starts putting bytes
# on the wire.
"-w", "%{size_upload} %{time_total} %{time_appconnect}", url]
def _parallel_upload(url: str, per_stream: int, streams: int, timeout: float):
"""Sum of concurrent upload stream rates.
The download learned in 0.1.x that one TCP stream cannot fill a fast line —
a single-stream check read this 450 Mbps connection as 54 — and grew
`_parallel_download` for it. The upload never did, in either the hourly
check or the peak, so both were reading one stream's ceiling and calling
it the line. Measured here: the same 2 MB carried by four streams instead
of one read 36% higher, and 4 MB over four streams read more than twice
what the shipping 2 MB over one did.
Every stream is handed the same immutable buffer, so the memory cost is
one stream's worth of zeros rather than N.
"""
if not shutil.which("curl"):
return None, 0
body = b"\0" * per_stream
procs = []
for _ in range(streams):
try:
spawned = time.monotonic()
procs.append((subprocess.Popen(
_upload_argv(url, timeout), stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL), spawned))
except OSError:
pass
# A pipe holds far less than a stream's body, so writing them in turn
# would serialise the very thing being parallelised: each child gets a
# thread that feeds it and collects its result.
outs = [None] * len(procs)
def feed(i, proc):
try:
outs[i] = proc.communicate(input=body, timeout=timeout + 10)[0]
except subprocess.TimeoutExpired:
proc.kill()
proc.wait()
workers = [threading.Thread(target=feed, args=(i, proc), daemon=True)
for i, (proc, _) in enumerate(procs)]
for w in workers:
w.start()
for w in workers:
w.join(timeout + 15)
windows, total_bytes = [], 0
for i, (proc, spawned) in enumerate(procs):
if proc.returncode != 0 or not outs[i]:
continue
try:
size, t_total, t_app = (float(x) for x in outs[i].decode().split())
except (ValueError, UnicodeDecodeError):
continue
total_bytes += int(size)
windows.append((size, spawned + t_app, spawned + t_total))
return _aggregate_rate(windows), total_bytes
def _curl_timed_upload(url: str, n_bytes: int, timeout: float):
"""One upload stream. Kept for the peak's estimate pass, which only needs
a rough rate to size the real one."""
if not shutil.which("curl"):
return None, 0
cmd = ["curl", "-fsS", "--proto", "=https", "--max-time", str(int(timeout)),
"-o", "/dev/null", "-X", "POST", "--data-binary", "@-",
"-H", "Content-Type: application/octet-stream",
# Not time_starttransfer: on a POST that lands part way through
# the body, not after it. The TLS handshake completing is when
# this request starts putting bytes on the wire.
"-w", "%{size_upload} %{time_total} %{time_appconnect}", url]
try:
r = subprocess.run(cmd, input=b"\0" * n_bytes, capture_output=True,
timeout=timeout + 5, check=False)
except (subprocess.TimeoutExpired, OSError):
return None, 0
if r.returncode != 0:
return None, 0
try:
size, t_total, t_app = (float(x) for x in r.stdout.decode().split())
except (ValueError, UnicodeDecodeError):
return None, 0
return _rate_over_payload(size, t_total, t_app), int(size)
def _aggregate_rate(windows):
"""Mbps carried by a set of parallel streams.
NOT the sum of their individual rates. Streams do not start or finish
together — TLS handshakes complete tens to hundreds of milliseconds apart
— so a stream that outlives the others has the line to itself and measures
all of it. Adding that to what its siblings measured while sharing counts
the same link two, three, four times. Observed on this ~450 Mbps line:
summing gave 647 / 629 / 538 Mbps for transfers that actually carried
323 / 276 / 269.
The honest figure is what crossed the wire divided by the time the wire
spent carrying it: total bytes over the union of the streams' payload
windows. The union rather than first-start-to-last-finish, so a gap
between streams is not billed as throughput.
`windows` is (bytes, absolute start, absolute end) per stream.
"""
windows = [w for w in windows if w[0] > 0 and w[2] > w[1]]
if not windows:
return None
total_bytes = sum(w[0] for w in windows)
spans = sorted((w[1], w[2]) for w in windows)
union, cur_s, cur_e = 0.0, spans[0][0], spans[0][1]
for s, e in spans[1:]:
if s > cur_e:
union += cur_e - cur_s
cur_s, cur_e = s, e
else:
cur_e = max(cur_e, e)
union += cur_e - cur_s
if union < MIN_TIMED_WINDOW_S:
return None
return total_bytes * 8 / 1e6 / union
def _parallel_download(url: str, streams: int, timeout: float):
"""Sum of concurrent stream rates.
One TCP stream at ~10 ms of latency tops out far below a fast line's
capacity — a single-stream check read this 450 Mbps connection as 54.
Real page loads and video players open several connections, so several
streams is the honest simulation, and their sum is the number.
"""
if not shutil.which("curl"):
return None, 0
procs = []
for _ in range(streams):
try:
# curl times everything from its own start, and the children are
# spawned a few milliseconds apart, so their clocks have to be
# put on a common origin before their windows can be compared.
spawned = time.monotonic()
procs.append((subprocess.Popen(
["curl", "-fsS", "--proto", "=https",
"--max-time", str(int(timeout)), "-o", "/dev/null",
"-w", "%{size_download} %{time_total} %{time_starttransfer}",
url],
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True),
spawned))
except OSError:
pass
windows, total_bytes = [], 0
for p, spawned in procs:
try:
out, _ = p.communicate(timeout=timeout + 10)
except subprocess.TimeoutExpired:
p.kill()
p.wait()
continue
if p.returncode != 0:
continue
try:
size, t_total, t_start = (float(x) for x in out.split())
except ValueError:
continue
total_bytes += int(size)
windows.append((size, spawned + t_start, spawned + t_total))
return _aggregate_rate(windows), total_bytes
# Each stream aims for about this much time actually carrying bytes: long
# enough that TCP's ramp-up is a small share of what is timed, short enough
# that the check stays something nobody notices.
CONTENT_TARGET_S = 0.5
# A stream never goes below this, so a hint that came in low cannot shrink
# the next transfer into a degenerate one.
CONTENT_STREAM_FLOOR = 500_000
# And never above this, which is what bounds the hourly data budget. Only a
# fast line reaches either cap; everything slower asks for less and gets it.
CONTENT_DOWN_STREAM_CAP = 3_000_000
CONTENT_UP_STREAM_CAP = 2_000_000
def content_stream_bytes(hint_mbps, streams: int, cap: int) -> int:
"""Bytes for one stream: about CONTENT_TARGET_S of payload at the rate
this line last showed, bounded both ways.
A fixed size cannot serve both ends of the range it has to. Twelve MB is
a quarter of a second on a fast line and nine seconds of a saturated link
on a 10 Mbps one — the users least able to spare it were paying the most
for it, hourly. Sizing by time inverts that: the cap is reached only by
lines that can afford it, and a slow line asks for a fraction.
With no hint — the first check on a network — the cap is what it sends,
because there is nothing yet to size against and one honest measurement
is what produces the hint for every check after it.
"""
if not hint_mbps or hint_mbps <= 0:
return cap
per_stream_mbps = float(hint_mbps) / max(1, streams)
want = int(per_stream_mbps / 8 * CONTENT_TARGET_S * 1e6)
return max(CONTENT_STREAM_FLOOR, min(cap, want))
def content_test(down_hint_mbps=None, up_hint_mbps=None,
streams: int = 4) -> dict:
"""The scheduled check.
Both directions are carried by `streams` parallel connections, each sized
for a target duration rather than by dividing a fixed budget. Dividing a
budget was how the download bug worked from one side and the upload's from
the other: more streams meant shorter streams, and a stream too short to
time is withheld.
Costs up to ~20 MB on a line fast enough to reach both caps, and a
fraction of that below — about 2 MB on a 25 Mbps line, where the old fixed
16 MB took nine seconds of the link every hour.
"""
started = time.time()
per_stream = content_stream_bytes(down_hint_mbps, streams,
CONTENT_DOWN_STREAM_CAP)
up_per_stream = content_stream_bytes(up_hint_mbps, streams,
CONTENT_UP_STREAM_CAP)
down_mbps, down_n = _parallel_download(
CLOUDFLARE_DOWN.format(n=per_stream), streams, timeout=30)
if down_mbps is None and per_stream < CONTENT_DOWN_STREAM_CAP:
# Sized from history, and the line turned out to be faster than that
# history says — so fast that the streams finished inside the window
# too short to time, and were withheld. Nothing is stored for a
# withheld check, so the hint would never learn better and every
# check after this one would ask for the same too-short transfer and
# report nothing, forever. One pass at the cap re-anchors it.
retry_mbps, retry_n = _parallel_download(
CLOUDFLARE_DOWN.format(n=CONTENT_DOWN_STREAM_CAP), streams,
timeout=30)
down_mbps, down_n = retry_mbps, down_n + retry_n
up_mbps, up_n = _parallel_upload(CLOUDFLARE_UP, up_per_stream, streams,
timeout=30)
if up_mbps is None and up_per_stream < CONTENT_UP_STREAM_CAP:
retry_mbps, retry_n = _parallel_upload(
CLOUDFLARE_UP, CONTENT_UP_STREAM_CAP, streams, timeout=30)
up_mbps, up_n = retry_mbps, up_n + retry_n
return {
"kind": "content",
"engine": "cloudflare",
"ok": down_mbps is not None,
"down_mbps": round(down_mbps, 1) if down_mbps else None,
"up_mbps": round(up_mbps, 1) if up_mbps else None,
"bytes": down_n + up_n,
"started": started,
"ended": time.time(),
}
def _peak_ookla() -> Optional[dict]:
"""The official Speedtest CLI, when the user has installed it."""
if not shutil.which("speedtest"):
return None
try:
r = subprocess.run(
["speedtest", "--format=json", "--accept-license", "--accept-gdpr"],
capture_output=True, text=True, timeout=120, check=False)
except (subprocess.TimeoutExpired, OSError):
return None
if r.returncode != 0:
return None
try:
j = json.loads(r.stdout)
return {
"engine": "ookla",
"ok": True,
"down_mbps": round(j["download"]["bandwidth"] * 8 / 1e6, 1),
"up_mbps": round(j["upload"]["bandwidth"] * 8 / 1e6, 1),
"ping_idle": round(j["ping"]["latency"], 1),
"jitter": round(j["ping"].get("jitter", 0), 1),
"bytes": j["download"].get("bytes", 0) + j["upload"].get("bytes", 0),
"server": f'{j["server"].get("name", "")} · {j["server"].get("location", "")}',
"url": j.get("result", {}).get("url", ""),
}
except (ValueError, KeyError, TypeError, AttributeError):
# Someone else's JSON: a missing key, a string where a number was
# expected, a list where an object was. Any of those is a failed
# engine, not a dead worker thread.
return None
PEAK_TARGET_S = 10 # aim each sustained pass at about this long
PEAK_STREAMS = 4
# __down 403s any single request of 100 MB or more; each parallel stream
# stays under that and the streams together still carry a fast line.
CLOUDFLARE_DOWN_MAX = 99_999_999
PEAK_DOWN_FLOOR = 10_000_000
PEAK_UP_FLOOR = 5_000_000
PEAK_UP_CAP = 100_000_000 # also bounds the in-memory upload body
def _sized_pass(mbps: float, floor: int, cap: int) -> int:
"""Bytes that should take about PEAK_TARGET_S at the measured rate."""
return max(floor, min(cap, int(mbps / 8 * PEAK_TARGET_S * 1e6)))
def _pass_seconds(mbps: float, n_bytes: int) -> float:
return n_bytes * 8 / (mbps * 1e6)
def _peak_cloudflare() -> Optional[dict]:
"""An estimate pass sizes a sustained pass.
Fixed sizes made the whole test finish inside TCP ramp-up on a fast
line (2-3 s end to end), which both under-reads the line and leaves
the loaded-latency window with a handful of probe samples. The
estimate pass measures the rate; the sustained pass is sized to hold
that rate for ~PEAK_TARGET_S, split over parallel streams because a
single stream can neither exceed the per-request byte cap nor fill a
fast line by itself. A slow line's estimate pass already runs that
long and doubles as the sustained pass.
"""
total = 0
best_down, size = _curl_timed_download(CLOUDFLARE_DOWN.format(n=25_000_000),
timeout=40)
total += size
if best_down and _pass_seconds(best_down, size) < PEAK_TARGET_S * 0.6:
n = _sized_pass(best_down / PEAK_STREAMS, PEAK_DOWN_FLOOR,
CLOUDFLARE_DOWN_MAX)
mbps, size = _parallel_download(CLOUDFLARE_DOWN.format(n=n),
PEAK_STREAMS, timeout=40)
total += size
if mbps:
best_down = max(best_down, mbps)
best_up = 0.0
up_est, size = _curl_timed_upload(CLOUDFLARE_UP, 10_000_000, timeout=40)
total += size
if up_est:
best_up = up_est
if _pass_seconds(up_est, size) < PEAK_TARGET_S * 0.6:
# Per stream, as the download pass already sizes itself: the same
# total goes up, split four ways, so this costs no more data than
# the single stream it replaces and stops reading one stream's
# ceiling as the line.
n = _sized_pass(up_est / PEAK_STREAMS, PEAK_UP_FLOOR // PEAK_STREAMS,
PEAK_UP_CAP // PEAK_STREAMS)
mbps, size = _parallel_upload(CLOUDFLARE_UP, n, PEAK_STREAMS,
timeout=40)
total += size
if mbps:
best_up = max(best_up, mbps)
if not best_down:
return None
return {
"engine": "cloudflare",
"ok": True,
"down_mbps": round(best_down, 1),
"up_mbps": round(best_up, 1) if best_up else None,
"bytes": total,
"server": "speed.cloudflare.com",
}
def _peak_fast() -> Optional[dict]:
"""Download-only, via the Netflix OCA endpoints fast.com hands out.
The API picks the hosts, so each one is vetted before it is fetched
(see vet_target) and a target that does not pass is skipped rather
than failing the test — a bad entry in someone else's JSON should
cost us one candidate, not the measurement.
"""
r = _curl([FAST_API], timeout=15)
if not r or r.returncode != 0:
return None
try:
targets = [t["url"] for t in json.loads(r.stdout).get("targets", []) if t.get("url")]
except (ValueError, KeyError, TypeError, AttributeError):
return None
vetted = [v for v in (vet_target(u) for u in targets if isinstance(u, str)) if v]
best = 0.0
total = 0
for url, resolve in vetted[:3]:
mbps, size = _curl_timed_download(url, timeout=30, resolve=resolve)
total += size
if mbps:
best = max(best, mbps)
if not best:
return None
return {"engine": "fast.com", "ok": True, "down_mbps": round(best, 1),
"up_mbps": None, "bytes": total, "server": "Netflix OCA"}
def peak_test(engine: str = "Auto") -> dict:
"""On-demand, engine per the user's setting."""
started = time.time()
order = {
"Auto": (_peak_ookla, _peak_cloudflare, _peak_fast),
"Ookla": (_peak_ookla,),
"Cloudflare": (_peak_cloudflare,),
"fast.com": (_peak_fast,),
}.get(engine, (_peak_ookla, _peak_cloudflare, _peak_fast))
for fn in order:
result = fn()
if result:
result.update({"kind": "peak", "started": started, "ended": time.time()})
return result
return {"kind": "peak", "engine": engine, "ok": False,
"started": started, "ended": time.time()}
plugins/io.github.x3me.nexthop/nexthopd/state.py
+149
View File
@@ -0,0 +1,149 @@
"""Reading and writing the JSON state files, safely.
live.json is rewritten twice a second and is all the bar widget ever looks
at; recent.json is a pre-downsampled 30-minute window so the panel's default
graphs paint without a query; apps.json is per-application traffic. All are
written to a temp file and renamed, so a reader never sees a half-written
file. The QML side does not open any of them itself (0.1.9): `nexthop
stream` reads them through `read_text_bounded` — no symlink following, a
regular file or nothing, a size cap on the read itself — and hands the shell
one re-serialised line per record.
"""
import json
import os
import stat
import tempfile
import time
from pathlib import Path
from .paths import APPS, LIVE, RECENT
def write_atomic(path: Path, payload: dict, durable: bool = False):
"""Write a JSON file so a reader sees the old one or the new one.
The temp file plus rename is what gives readers that guarantee, and it
costs nothing. The fsync is a different promise — that the bytes
survive a power cut — and none of the snapshots written here needs
it: each is replaced within seconds of the daemon starting. On btrfs
that fsync was 25× the payload at the block layer (62.5 KiB per 2.5 KB
write, measured), twice a second. `durable` keeps it for a caller
that genuinely wants it.
"""
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=f".{path.name}.")
try:
with os.fdopen(fd, "w") as f:
json.dump(payload, f, separators=(",", ":"))
f.flush()
if durable:
os.fsync(f.fileno())
os.replace(tmp, path)
except BaseException:
try:
os.unlink(tmp)
except OSError:
pass
raise
def read_text_bounded(path: Path, max_bytes: int):
"""Read a state file, enforcing every property on the fd actually read.
`O_NOFOLLOW` refuses a symlinked path outright, `O_NONBLOCK` means a
FIFO left at the path returns instead of stalling the caller, `fstat`
on the descriptor proves it is a regular file, and the cap bounds the
read itself rather than trusting a size sampled beforehand. Returns
(text, stamp) or None; `stamp` is (mtime_ns, size), enough for a
caller to skip re-reading an unchanged file.
This is the only way state reaches a reader — the QML side consumes it
through `nexthop stream` rather than opening these paths itself, so an
oversized or non-regular file can never allocate or block inside the
long-lived shell process.
"""
try:
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK)
except OSError:
return None
try:
st = os.fstat(fd)
if not stat.S_ISREG(st.st_mode):
return None
chunks, total = [], 0
while total <= max_bytes:
try:
chunk = os.read(fd, min(65536, max_bytes + 1 - total))
except BlockingIOError:
break
except OSError:
return None
if not chunk:
break
chunks.append(chunk)
total += len(chunk)
if total > max_bytes:
return None
stamp = (st.st_mtime_ns, st.st_size)
finally:
os.close(fd)
try:
return b"".join(chunks).decode("utf-8"), stamp
except UnicodeDecodeError:
return None
def read_json(path: Path, default=None, max_bytes: int = 4 * 1024 * 1024):
"""Bounded read of a state file, parsed. The bound lives on the read,
not on a prior stat, for the same reason as the daemon's config read."""
got = read_text_bounded(path, max_bytes)
if got is None:
return default
try:
return json.loads(got[0])
except ValueError:
return default
def retire_legacy_snapshots(old_dir: Path, new_dir: Path, now: float = None):
"""The snapshots moved to the runtime dir in 0.2.22; tidy the old place.
`nexthop stream` is a long-lived process that resolved its paths when it
started, so a reader from before the move keeps watching the state dir
for as long as it lives — through the daemon handover, until the shell
reloads the QML. Deleting live.json there would leave that reader
holding the last number it saw, as if it were current. It is rewritten
once instead, as a tombstone: state "no-daemon", no index, and no pid,
so the old bar shows "no data" and the old version watch finds nothing
to retire. recent.json and apps.json are simply removed.
"""
old_dir, new_dir = Path(old_dir), Path(new_dir)
if old_dir == new_dir:
return
now = time.time() if now is None else now
for name in (RECENT, APPS):
try:
(old_dir / name).unlink()
except OSError:
pass
live = old_dir / LIVE
got = read_text_bounded(live, 256 * 1024)
if got is None:
return
try:
payload = json.loads(got[0])
except ValueError:
payload = None
if not isinstance(payload, dict):
try:
live.unlink()
except OSError:
pass
return
for key in ("pid", "pid_start", "daemon_version"):
payload.pop(key, None)
payload.update({"t": round(now, 3), "state": "no-daemon", "index": None,
"band": None, "down_since": None})
write_atomic(live, payload, durable=True)
plugins/io.github.x3me.nexthop/nexthopd/store.py
+402
View File
@@ -0,0 +1,402 @@
"""Persistence: per-minute rows, hourly rollups, tests and events.
sqlite from the standard library, in WAL mode so the CLI can read a window
of history while the daemon is mid-write. Raw half-second samples never
reach the disk — they are folded into a minute row and discarded, which is
what keeps a month of continuous monitoring under about 12 MB.
"""
import functools
import sqlite3
import threading
import time
from pathlib import Path
from .probes import nearest_rank
SAMPLE_COLUMNS = [
"local_p50", "local_p95", "local_jitter", "local_loss",
"wan_p50", "wan_p95", "wan_jitter", "wan_loss",
# 0.2.20: the two order statistics the scored fold and its critics
# actually turn on. Lag leans on p75; Orb headlines a high-water max;
# LibreQoS takes a phase percentile. Comparing those against our own
# history was only possible as an upper bound because neither was ever
# written down — p50 and p95 alone cannot reconstruct either. Recorded
# now, scored never: the decision needs real days behind it, the same
# rule loaded latency was held to in 0.1.11.
"local_p75", "local_max", "wan_p75", "wan_max",
"lag", "rx_bps", "tx_bps", "signal_dbm",
"resp", "rel", "spd", "idx",
# Latency split by what the link was doing at the time. The gap between
# them is bufferbloat, and it only accumulates into something worth
# scoring if it is recorded minute by minute first.
"lag_idle", "lag_loaded",
# 0.2.0: what ICMP alone would have scored, beside the instrument-
# scored lag — the basis switch stays auditable per minute.
"lag_icmp",
]
# Minute-only, and deliberately not in SAMPLE_COLUMNS: `rollup_hours` averages
# everything in that list, and a mean of drains destroys the one thing the
# drain is being stored for. Its value is quantised by probe cadence, so what
# has to survive is the DISTRIBUTION — sixty of them averaged is a number with
# none of that in it.
#
# 0.2.37. Until now the drain was published to live.json and stored nowhere,
# so a figure on screen could never be checked afterwards; the distribution
# that showed it was quantised had to come from the other implementation
# because this one had no history to look at.
#
# Three numbers rather than one. `drain_settled` says whether the link
# recovered or the window merely ended, so a censored floor is not read as a
# measurement. `drain_min_ms` is the tightest bound across the seated
# instruments beside the loosest, which is what is published today — carrying
# both is what lets the choice between them be settled from history instead of
# argued. `drain_src` names the instrument the published value came from,
# because each instrument's value is floored at its own cadence and knowing
# which one won is the difference between an auditable figure and a guess.
MINUTE_ONLY_REAL = ["drain_ms", "drain_min_ms", "drain_settled"]
MINUTE_ONLY_TEXT = ["drain_src"]
_COLS_SQL = ", ".join(f"{c} REAL" for c in SAMPLE_COLUMNS)
_MINUTE_EXTRA_SQL = ", ".join(
[f"{c} REAL" for c in MINUTE_ONLY_REAL] + [f"{c} TEXT" for c in MINUTE_ONLY_TEXT])
SCHEMA = f"""
CREATE TABLE IF NOT EXISTS minute (
ts INTEGER PRIMARY KEY, {_COLS_SQL}, {_MINUTE_EXTRA_SQL},
iface TEXT, network TEXT, probes TEXT
);
CREATE TABLE IF NOT EXISTS hour (
ts INTEGER PRIMARY KEY, {_COLS_SQL}, iface TEXT, network TEXT
);
CREATE TABLE IF NOT EXISTS tests (
ts INTEGER PRIMARY KEY, kind TEXT, engine TEXT,
down_mbps REAL, up_mbps REAL, ping_idle REAL, ping_loaded REAL,
jitter REAL, bytes INTEGER, server TEXT, ok INTEGER, detail TEXT,
network TEXT
);
CREATE TABLE IF NOT EXISTS events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ts INTEGER NOT NULL, ended_ts INTEGER, kind TEXT, severity TEXT,
leg TEXT, detail TEXT
);
CREATE INDEX IF NOT EXISTS events_ts ON events(ts);
CREATE INDEX IF NOT EXISTS tests_kind_ts ON tests(kind, ts);
"""
def _locked(method):
"""Serialise access to the one connection — see Store."""
@functools.wraps(method)
def wrapper(self, *args, **kwargs):
with self._lock:
return method(self, *args, **kwargs)
return wrapper
class Store:
"""One connection, one lock.
Three threads reach this object: the daemon loop (minute rows, events,
a read for Reliability on every tick), the content-test worker and the
peak-test worker (one row each when they finish). The connection is
opened with `check_same_thread=False`, which only tells the sqlite3
module to allow that — it does not make concurrent use of one
connection safe, and a forced overlap reproduces "bad parameter or
other API misuse" and a lost row. Every transaction here is a few
milliseconds, so a mutex is the whole fix; a writer thread with a
queue was considered and is more machinery than three callers need.
"""
def __init__(self, path: Path, read_only: bool = False):
self.path = Path(path)
self._lock = threading.RLock()
if read_only:
uri = f"file:{self.path}?mode=ro"
self.db = sqlite3.connect(uri, uri=True, timeout=5.0)
else:
self.path.parent.mkdir(parents=True, exist_ok=True)
self.db = sqlite3.connect(self.path, timeout=5.0,
check_same_thread=False)
self.db.executescript(SCHEMA)
self._migrate()
self.db.execute("PRAGMA journal_mode=WAL")
self.db.execute("PRAGMA synchronous=NORMAL")
self.db.commit()
self.db.row_factory = sqlite3.Row
def _migrate(self):
"""Additive migrations for databases created by older versions."""
for table, column in (("tests", "network"),
("minute", "lag_idle"), ("minute", "lag_loaded"),
("hour", "lag_idle"), ("hour", "lag_loaded"),
("minute", "lag_icmp"), ("hour", "lag_icmp"),
("minute", "probes"),
("minute", "local_p75"), ("hour", "local_p75"),
("minute", "local_max"), ("hour", "local_max"),
("minute", "wan_p75"), ("hour", "wan_p75"),
("minute", "wan_max"), ("hour", "wan_max"),
("minute", "drain_ms"),
("minute", "drain_min_ms"),
("minute", "drain_settled"),
("minute", "drain_src")):
try:
self.db.execute(
f"ALTER TABLE {table} ADD COLUMN {column} "
f"{'TEXT' if column in ('network', 'probes', 'drain_src') else 'REAL'}")
except sqlite3.OperationalError:
pass # column already there
@_locked
def close(self):
try:
self.db.close()
except sqlite3.Error:
pass
# ---------------------------------------------------------------- writes
@_locked
def put_minute(self, ts: int, values: dict, iface: str = "",
network: str = "", probes: str = ""):
extra = MINUTE_ONLY_REAL + MINUTE_ONLY_TEXT
cols = ["ts"] + SAMPLE_COLUMNS + extra + ["iface", "network", "probes"]
row = ([int(ts)]
+ [values.get(c) for c in SAMPLE_COLUMNS]
+ [values.get(c) for c in extra]
+ [iface, network, probes])
placeholders = ", ".join("?" * len(cols))
self.db.execute(
f"INSERT OR REPLACE INTO minute ({', '.join(cols)}) VALUES ({placeholders})",
row,
)
self.db.commit()
@_locked
def put_test(self, ts: int, kind: str, engine: str, **kw):
self.db.execute(
"""INSERT OR REPLACE INTO tests
(ts, kind, engine, down_mbps, up_mbps, ping_idle, ping_loaded,
jitter, bytes, server, ok, detail, network)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)""",
(int(ts), kind, engine, kw.get("down_mbps"), kw.get("up_mbps"),
kw.get("ping_idle"), kw.get("ping_loaded"), kw.get("jitter"),
kw.get("bytes"), kw.get("server"), 1 if kw.get("ok", True) else 0,
kw.get("detail", ""), kw.get("network", "")),
)
self.db.commit()
@_locked
def open_event(self, ts: int, kind: str, severity: str, leg: str, detail: str) -> int:
cur = self.db.execute(
"INSERT INTO events (ts, kind, severity, leg, detail) VALUES (?,?,?,?,?)",
(int(ts), kind, severity, leg, detail),
)
self.db.commit()
return cur.lastrowid
@_locked
def close_event(self, event_id: int, ended_ts: int, detail: str = None):
if detail is None:
self.db.execute("UPDATE events SET ended_ts=? WHERE id=?",
(int(ended_ts), event_id))
else:
self.db.execute("UPDATE events SET ended_ts=?, detail=? WHERE id=?",
(int(ended_ts), detail, event_id))
self.db.commit()
# ------------------------------------------------------------- maintenance
@_locked
def rollup_hours(self, now: float = None):
"""Fold complete minutes into hour rows.
Averages the averages, which is fair because every minute row covers
the same span. Percentiles do not survive that — an hourly p95 built
from sixty per-minute p95s is a mean of p95s, and it is labelled as
such wherever it is displayed. The same caveat binds harder to the
new max columns: an hourly `local_max` is a mean of sixty maxima,
which is not the hour's worst sample and must never be shown as one.
Use the minute rows for anything that reasons about the tail.
An hour that spanned two networks is labelled with neither: MAX()
would pick whichever name sorts last and file the other network's
minutes under it. Blank is "mixed", which no consumer can mistake
for a network.
"""
now = now or time.time()
current_hour = int(now // 3600) * 3600
avg = ", ".join(f"AVG({c}) AS {c}" for c in SAMPLE_COLUMNS)
self.db.execute(
f"""INSERT OR REPLACE INTO hour
(ts, {', '.join(SAMPLE_COLUMNS)}, iface, network)
SELECT (ts / 3600) * 3600 AS bucket, {avg},
CASE WHEN COUNT(DISTINCT iface) > 1 THEN ''
ELSE MAX(iface) END,
CASE WHEN COUNT(DISTINCT network) > 1 THEN ''
ELSE MAX(network) END
FROM minute WHERE ts < ? GROUP BY bucket""",
(current_hour,),
)
self.db.commit()
@_locked
def prune(self, minute_days: int = 7, hour_days: int = 400, now: float = None):
now = now or time.time()
self.db.execute("DELETE FROM minute WHERE ts < ?",
(int(now - minute_days * 86400),))
self.db.execute("DELETE FROM hour WHERE ts < ?",
(int(now - hour_days * 86400),))
# Events were never pruned before 0.2.21 — about sixty rows a day,
# unbounded. They keep the hourly history's horizon.
self.db.execute("DELETE FROM events WHERE ts < ?",
(int(now - hour_days * 86400),))
self.db.commit()
@_locked
def close_orphans(self, now: float = None) -> int:
"""Close events a previous daemon left open. Returns how many.
Only the daemon that opened an event can close it, so one that
died mid-outage — or was retired by the version handover with a
rate-drop open — leaves `ended_ts` NULL for good. Two readers
treat NULL as "still happening": `outage_stats` would charge such
an outage against every Reliability window forever, and `events`
would list it as ongoing. When it actually ended is unknowable,
so it is closed at the shortest span the store accepts rather
than at a guessed later time: undercharging by the lost tail is
the safe direction, and inventing a duration is not.
Called once, after the lock is held — a second daemon that loses
the flock must not close the running one's events on its way out.
"""
cur = self.db.execute(
"UPDATE events SET ended_ts = ts + 1 WHERE ended_ts IS NULL")
self.db.commit()
return cur.rowcount
# ---------------------------------------------------------------- reads
@_locked
def series(self, seconds: float, now: float = None,
resolution: str = "auto") -> list:
"""History over a window, at whichever resolution suits it.
Auto: under six hours reads per-minute rows; anything longer reads
hourly ones, so a seven-day graph is 168 points rather than 10,080.
Callers that genuinely want the fine rows (the 24 h experience
ribbon) ask for "minute" explicitly.
"""
now = now or time.time()
if resolution in ("minute", "hour"):
table = resolution
else:
table = "minute" if seconds <= 6 * 3600 else "hour"
rows = self.db.execute(
f"SELECT * FROM {table} WHERE ts >= ? ORDER BY ts",
(int(now - seconds),),
).fetchall()
return [dict(r) for r in rows], table
@_locked
def tests(self, limit: int = 20, kind: str = None) -> list:
if kind:
rows = self.db.execute(
"SELECT * FROM tests WHERE kind=? ORDER BY ts DESC LIMIT ?",
(kind, limit)).fetchall()
else:
rows = self.db.execute(
"SELECT * FROM tests ORDER BY ts DESC LIMIT ?", (limit,)).fetchall()
return [dict(r) for r in rows]
@_locked
def events(self, seconds: float = 7 * 86400, limit: int = 100,
now: float = None) -> list:
"""Events overlapping the window, newest first.
Filtering on start time alone dropped an outage that began before
the window and ended inside it — the one the user opens the list
to see. Same overlap rule `outage_stats` has always used.
"""
now = now or time.time()
start = int(now - seconds)
rows = self.db.execute(
"""SELECT * FROM events
WHERE ts >= ? OR ended_ts IS NULL OR ended_ts >= ?
ORDER BY ts DESC LIMIT ?""",
(start, start, limit)).fetchall()
return [dict(r) for r in rows]
@_locked
def baseline_speed(self, days: int = 30, network: str = "",
min_samples: int = 5, now: float = None,
fallback: bool = True):
"""This connection's own normal: the p90 of recent content downloads.
p90 rather than max so one lucky quiet-hour run does not set a bar
the line can never reach again. Scoped to the current network when
it has enough samples — the office's normal is not the home's —
falling back to all networks, and to None until there is enough
history to mean anything.
"""
now = now or time.time()
since = int(now - days * 86400)
def p90(rows):
vals = sorted(r["down_mbps"] for r in rows
if r["down_mbps"] is not None)
if len(vals) < min_samples:
return None
return nearest_rank(vals, 0.9)
if network:
rows = self.db.execute(
"""SELECT down_mbps FROM tests
WHERE kind='content' AND ok=1 AND ts >= ? AND network = ?""",
(since, network)).fetchall()
result = p90(rows)
if result is not None:
return result
# The caller decides whether a cross-network baseline is meaningful.
# For the degradation penalty it is not: "is it normal here" cannot
# be answered with another network's normal.
if not fallback:
return None
rows = self.db.execute(
"""SELECT down_mbps FROM tests
WHERE kind='content' AND ok=1 AND ts >= ?""",
(since,)).fetchall()
return p90(rows)
@_locked
def outage_stats(self, seconds: float, now: float = None):
"""(fraction fully down, count of disruptions, fraction disrupted).
Disruptions carry their duration as well as their count because
reliability charges both kinds of interruption in the same currency —
time. Counting alone made three brief blips outweigh an hour offline.
"""
now = now or time.time()
start = now - seconds
rows = self.db.execute(
"""SELECT ts, ended_ts, kind FROM events
WHERE kind IN ('outage', 'disruption') AND (ended_ts IS NULL OR ended_ts >= ?)""",
(int(start),)).fetchall()
down = 0.0
disrupted = 0.0
disruptions = 0
for r in rows:
begin = max(r["ts"], start)
end = r["ended_ts"] if r["ended_ts"] else now
end = min(end, now)
if end <= begin:
continue
if r["kind"] == "outage":
down += end - begin
else:
disruptions += 1
disrupted += end - begin
span = seconds if seconds else 0.0
return ((down / span if span else 0.0), disruptions,
(disrupted / span if span else 0.0))
plugins/io.github.x3me.nexthop/nexthopd/update.py
+246
View File
@@ -0,0 +1,246 @@
"""Is a newer version published? Notify, never install.
Omarchy checks itself for updates (`omarchy-update-available` looks at its own
checkout and its package) but nothing checks plugins, so a user can sit on an
old Nexthop indefinitely without ever being told. This closes that gap the
smallest way it can be closed.
**This module never updates anything.** It answers one question — is the
installed checkout behind its origin — and the answer becomes a quiet glyph in
the panel naming the command the user can run. Updating stays where Omarchy
put it: `omarchy plugin update`, which shows the diff and asks. A plugin that
fetched and ran new code would be self-modifying code inside a system that
deliberately gates updates behind human review, and it would reopen a security
review that took four rounds to clear.
How it stays cheap and read-only:
* `git ls-remote` asks the remote for its HEAD without writing a single byte
into the user's checkout — no fetch, no new objects, no refs touched. It
takes well under a second and needs no credentials.
* Whether we are *behind* rather than merely *different* is then decided from
objects we already have, so a developer checkout that is ahead of origin is
never nagged.
* Egress is to the repository the user installed from and nowhere else. It
carries nothing about them or their network. It is still egress, so it is
disclosed and `updateCheck` turns it off.
The one piece of untrusted input here is the commit id the remote hands back,
and it goes on to be an argument to another `git` call — so it is validated
against the exact 40-hex shape before it reaches a subprocess, the same
doctrine `vet_target()` applies to URLs we did not choose.
"""
import os
import re
import shutil
import subprocess
import threading
from pathlib import Path
# A git object id and nothing else. This is the guard that matters: the value
# arrives from the network and is then passed as an argument to git.
RE_SHA = re.compile(r"^[0-9a-f]{40}$")
# A release is a rare event, and the check exists to catch the user who would
# otherwise never look. Daily is generous.
CHECK_INTERVAL_S = 24 * 3600
# Not at startup: the daemon restarts with the shell, and a check on every
# restart would be noise for no benefit. Nothing is lost by waiting.
FIRST_CHECK_DELAY_S = 300
# The remote may be slow, unreachable, or a captive portal that answers
# everything. None of those may stall the daemon.
GIT_TIMEOUT_S = 20
# `ls-remote` prints one short line per ref; this is far past HEAD alone.
MAX_LS_REMOTE_BYTES = 64 * 1024
def verdict(local: str, remote: str, have_remote: bool,
head_before_remote: bool, remote_before_head: bool) -> str:
"""Where the checkout stands relative to origin. Pure, so it is testable.
* `current` — the same commit.
* `behind` — origin is strictly ahead of us: an update.
* `ahead` — we are strictly ahead of origin: a dev checkout.
* `diverged` — neither contains the other.
* `unknown` — we could not tell, and say so rather than guessing.
Both ancestry directions are needed, and the reason is the likeliest
case of all: `omarchy plugin update` fetches before it shows its diff, so
a user who looked and said "not now" already *holds* origin's commit
while still being behind it. Deciding "behind" from "we have never seen
that object" alone would show that user nothing.
"""
if not local or not remote:
return "unknown"
if local == remote:
return "current"
if not have_remote:
# We do not hold origin's commit at all, so it is newer than anything
# we know about.
return "behind"
if head_before_remote:
return "behind"
if remote_before_head:
return "ahead"
return "diverged"
class UpdateWatch:
"""Asks origin, on a slow cadence, whether this checkout is behind.
The result lives in memory only. A daemon restart forgets it and waits
`FIRST_CHECK_DELAY_S` before asking again, which is why no fifth state
file was added for this.
"""
def __init__(self, repo: Path = None, enabled: bool = True, spawn=None):
# Derived from this file, not from the working directory: the daemon
# can be started from anywhere.
self.repo = Path(repo) if repo else Path(__file__).resolve().parent.parent
self.enabled = enabled
self.state = "unknown"
self.checked_ts = None
self._next = None
# How a check is run. Off the loop by default: `ls-remote` may sit
# at its 20 s timeout on exactly the flaky network where the
# outage watch matters, and the loop must not wait for it. Tests
# pass a synchronous spawn so the verdict lands within the tick.
self._spawn = spawn or self._in_thread
self._lock = threading.Lock()
self._inflight = False
self._pending = None # a verdict awaiting the next tick
@staticmethod
def _in_thread(fn):
threading.Thread(target=fn, name="update-check", daemon=True).start()
def _git(self, *args, capture: bool = True):
"""One git call. Fixed argv, no shell, bounded, always timed out."""
env = dict(os.environ)
# A credential prompt on a private or moved remote would otherwise
# block until the timeout every single time.
env["GIT_TERMINAL_PROMPT"] = "0"
env.pop("GIT_ASKPASS", None)
env.pop("SSH_ASKPASS", None)
# `git -C <dir>` does not require <dir> to BE a repository: git
# walks up until it finds one. A copy-install with no .git of its
# own, inside a home that is itself a checkout (dotfiles), would
# otherwise be compared against — and contact the remote of — the
# wrong repository every day. The ceiling stops the walk at our
# parent, so a missing .git answers "unknown", never someone else.
env["GIT_CEILING_DIRECTORIES"] = str(self.repo.parent)
try:
proc = subprocess.run(
["git", "-C", str(self.repo),
# No credential helper and no askpass program may run for
# this: a private or moved remote gets "unknown", not a
# keyring prompt once a day. GIT_TERMINAL_PROMPT above
# covers only git's own tty prompt.
"-c", "credential.helper=", "-c", "core.askPass=",
*args],
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE if capture else subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=GIT_TIMEOUT_S, env=env,
)
except (OSError, subprocess.TimeoutExpired):
return None, ""
out = ""
if capture and proc.stdout:
out = proc.stdout[:MAX_LS_REMOTE_BYTES].decode(
"utf-8", "replace").strip()
return proc.returncode, out
def _local_head(self):
code, out = self._git("rev-parse", "HEAD")
return out if code == 0 and RE_SHA.match(out) else None
def _remote_head(self):
"""Origin's HEAD, or None. Writes nothing into the checkout."""
code, out = self._git("ls-remote", "origin", "HEAD")
if code != 0 or not out:
return None
sha = out.split()[0] if out.split() else ""
# The guard: this value came off the network and is about to become a
# git argument. Anything but a bare object id is refused outright.
return sha if RE_SHA.match(sha) else None
def check(self) -> str:
"""Run one check now and return the verdict. Read-only throughout."""
if not shutil.which("git"):
return "unknown"
local = self._local_head()
if not local:
return "unknown" # not a git checkout, or no commits
remote = self._remote_head()
if not remote:
return "unknown" # offline, no origin, or a junk answer
if local == remote:
return "current"
# Do we already hold origin's commit? Only objects we have are
# consulted from here on, so no fetch is ever needed.
code, _ = self._git("cat-file", "-e", remote + "^{commit}",
capture=False)
have = code == 0
before, after = False, False
if have:
# Exit 1 means "not an ancestor", which is an answer, not an error.
code, _ = self._git("merge-base", "--is-ancestor", local, remote,
capture=False)
before = code == 0
code, _ = self._git("merge-base", "--is-ancestor", remote, local,
capture=False)
after = code == 0
return verdict(local, remote, have, before, after)
def _run_check(self):
state = self.check()
with self._lock:
self._pending = state
self._inflight = False
def _collect(self, now: float):
with self._lock:
state, self._pending = self._pending, None
if state is not None:
self.state = state
self.checked_ts = round(now)
def tick(self, now: float):
"""Called from the daemon loop; starts a check when one is due and
adopts the verdict of one that has finished. Never blocks."""
if not self.enabled:
# Turning the setting off clears any standing notice, so the
# glyph disappears rather than lingering with a stale answer.
# A check already running is left to finish and its answer
# dropped here, unread.
self.state, self.checked_ts, self._next = "unknown", None, None
with self._lock:
self._pending = None
return
self._collect(now)
if self._next is None:
self._next = now + FIRST_CHECK_DELAY_S
return
if now < self._next:
return
self._next = now + CHECK_INTERVAL_S
with self._lock:
if self._inflight:
return
self._inflight = True
self._spawn(self._run_check)
# A synchronous spawn has already finished; adopt it now rather
# than a tick later.
self._collect(now)
def snapshot(self) -> dict:
"""What the panel reads. None while nothing has been established."""
if not self.enabled or self.state == "unknown":
return None
return {
"state": self.state,
"available": self.state == "behind",
"checked_ts": self.checked_ts,
}