439 lines
22 KiB
Python
Executable File
439 lines
22 KiB
Python
Executable File
#!/usr/bin/python3
|
||
"""Read-only hardware sampling and bounded, persistent event summaries."""
|
||
import argparse
|
||
import csv
|
||
import datetime as dt
|
||
import fcntl
|
||
import hashlib
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import re
|
||
import subprocess
|
||
import time
|
||
|
||
BASE = Path('/var/lib/hardware-watch')
|
||
LOG = Path('/var/log/hardware-watch')
|
||
CONFIG = Path('/etc/hardware-watch.json')
|
||
VERSION = '1.0'
|
||
PATTERN = (r'NVRM|NVIDIA|AER:|PCIe Bus Error|BadTLP|BadDLLP|'
|
||
r'EDAC|Machine check|Hardware Error|mce:|'
|
||
r'I/O error|Buffer I/O|EXT4-fs error|BTRFS.*error|'
|
||
r'nvme.*(timeout|reset|fail|critical)|ata[0-9].*(error|fail|reset)|'
|
||
r'critical temperature|temperature above threshold|thermal.*(thrott|critical)')
|
||
NAMES = {
|
||
'power': '显卡辅助供电检测失败', 'gpu_init': '显卡驱动初始化失败',
|
||
'pcie_fatal': 'PCIe 不可纠正错误', 'pcie_correctable': 'PCIe 可纠正通信错误',
|
||
'gpu_xid': 'NVIDIA Xid 错误', 'memory_cpu': 'CPU／内存硬件报错',
|
||
'storage': '磁盘或存储通信错误', 'thermal': '硬件温度告警',
|
||
'gpu_missing': 'PCIe 显卡数量不足', 'gpu_driver': '显卡管理接口异常',
|
||
'gpu_identity': '预期显卡 UUID 缺失', 'gpu_temp': '显卡温度超过设定阈值',
|
||
'disk_missing': '预期磁盘缺失', 'smart': '磁盘 SMART 健康告警',
|
||
'sensor_temp': '传感器温度告警', 'journal': '系统日志采集异常',
|
||
'monitor': '硬件日志采集器', 'boot': '系统启动',
|
||
'clock': '系统时钟发生明显变化',
|
||
}
|
||
|
||
def stamp():
|
||
return dt.datetime.now().astimezone().isoformat(timespec='seconds')
|
||
|
||
def read(path, default=''):
|
||
try:
|
||
return Path(path).read_text().strip()
|
||
except OSError:
|
||
return default
|
||
|
||
def load(path, default):
|
||
try:
|
||
return json.loads(Path(path).read_text())
|
||
except (OSError, ValueError):
|
||
return default
|
||
|
||
def save(path, obj):
|
||
tmp = Path(str(path) + '.tmp')
|
||
tmp.write_text(json.dumps(obj, ensure_ascii=False, indent=2) + '\n')
|
||
os.chmod(tmp, 0o640)
|
||
os.replace(tmp, path)
|
||
|
||
def command(args, timeout=12):
|
||
try:
|
||
p = subprocess.run(args, text=True, stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE, timeout=timeout, env={**os.environ, 'LC_ALL': 'C'})
|
||
return p.returncode, p.stdout, p.stderr
|
||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||
return 124, '', str(exc)
|
||
|
||
def emit(kind, severity='INFO', **values):
|
||
row = {'observed_at': stamp(), 'severity': severity, 'category': kind,
|
||
'summary': NAMES.get(kind, kind), **values}
|
||
path = LOG / ('events-' + dt.datetime.now().astimezone().strftime('%Y-%m-%d') + '.jsonl')
|
||
with path.open('a', encoding='utf-8') as f:
|
||
f.write(json.dumps(row, ensure_ascii=False) + '\n')
|
||
os.chmod(path, 0o640)
|
||
|
||
def classify(message):
|
||
low = message.lower()
|
||
if 'necessary power cables' in low:
|
||
return 'power', 'ERROR'
|
||
if 'rminitadapter failed' in low or 'rm_init_adapter failed' in low:
|
||
return 'gpu_init', 'ERROR'
|
||
if 'no nvidia gpu found' in low:
|
||
return 'gpu_missing', 'WARN'
|
||
if 'fallen off' in low or re.search(r'\bXid\b', message):
|
||
return 'gpu_xid', 'ERROR'
|
||
if any(x in low for x in ('severity=uncorrectable', 'aer: device recovery failed', 'non-functional downstream link')):
|
||
return 'pcie_fatal', 'ERROR'
|
||
if 'severity=correctable' in low or 'badtlp' in low or 'baddllp' in low or re.search(r'\[\s*12\]\s+Timeout', message):
|
||
return 'pcie_correctable', 'WARN'
|
||
if re.search(r'machine check|hardware error|edac.*(\b[cu]e\b|error)|mce:.*(error|bank)', low):
|
||
return 'memory_cpu', 'ERROR'
|
||
if re.search(r'i/o error|buffer i/o|ext4-fs error|btrfs.*error|nvme.*(timeout|reset|fail|critical)|ata[0-9].*(error|fail)', low):
|
||
return 'storage', 'ERROR'
|
||
if re.search(r'critical temperature|temperature above threshold|thermal.*(thrott|critical)', low):
|
||
return 'thermal', 'WARN'
|
||
return None
|
||
|
||
def kernel_events(state, cfg):
|
||
args = ['journalctl', '--no-pager', '-o', 'json', '_TRANSPORT=kernel',
|
||
'--grep=' + PATTERN, '--case-sensitive=no', '--lines=+20000']
|
||
cursor = state.get('cursor')
|
||
args.append('--after-cursor=' + cursor if cursor else '--since=' + cfg.get('history_since', '7 days ago'))
|
||
code, out, err = command(args, 22)
|
||
if code not in (0, 1):
|
||
state['journal_error'] = err[:1000] or f'journalctl exit {code}'
|
||
emit('journal', 'ERROR', detail=err[:1000], action='下次继续重试；原始 journal 未删除。')
|
||
if 'cursor' in err.lower() and 'cursor' in state:
|
||
state.pop('cursor')
|
||
return
|
||
state.pop('journal_error', None)
|
||
state['journal_last_success'] = stamp()
|
||
entries = []
|
||
for line in out.splitlines():
|
||
try:
|
||
entries.append(json.loads(line))
|
||
except ValueError:
|
||
continue
|
||
# A journal with a backwards clock step can replay records before its cursor.
|
||
recent = state.get('recent_source_ids', [])
|
||
seen = set(recent)
|
||
def source_id(entry):
|
||
return hashlib.sha256(entry.get('__CURSOR', '').encode()).hexdigest()[:24]
|
||
if not recent and cursor and entries and entries[-1].get('__CURSOR') == cursor:
|
||
recent = [source_id(entry) for entry in entries]
|
||
seen = set(recent)
|
||
emit('clock', 'WARN', detail='检测到历史校时造成 journal 游标重放；已跳过此前处理过的记录，避免误计为新故障。')
|
||
groups = {}
|
||
rows = 0
|
||
for entry in entries:
|
||
rows += 1
|
||
state['cursor'] = entry.get('__CURSOR', state.get('cursor'))
|
||
token = source_id(entry)
|
||
if token in seen:
|
||
continue
|
||
seen.add(token)
|
||
recent.append(token)
|
||
message = entry.get('MESSAGE', '')
|
||
if not isinstance(message, str):
|
||
continue
|
||
result = classify(message)
|
||
if not result:
|
||
continue
|
||
category, severity = result
|
||
match = re.search(r'(?:[0-9a-f]{4}:)?[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]', message, re.I)
|
||
device = match.group(0) if match else ''
|
||
normalized = re.sub(r'0x[0-9a-f]+|\b[0-9]+\b', '#', message, flags=re.I)
|
||
key = (entry.get('_BOOT_ID', ''), category, device, normalized)
|
||
realtime = int(entry.get('__REALTIME_TIMESTAMP', 0)) / 1000000
|
||
when = dt.datetime.fromtimestamp(realtime).astimezone().isoformat(timespec='seconds')
|
||
observed = stamp()
|
||
monotonic = entry.get('__MONOTONIC_TIMESTAMP')
|
||
group = groups.setdefault(key, {'category': category, 'severity': severity,
|
||
'source': 'kernel', 'device': device, 'boot_id': entry.get('_BOOT_ID', ''),
|
||
'first_at': when, 'last_at': when, 'first_monotonic_us': entry.get('__MONOTONIC_TIMESTAMP'),
|
||
'last_monotonic_us': monotonic, 'first_observed_at': observed,
|
||
'last_observed_at': observed, 'count': 0, 'detail': message[:1600]})
|
||
group['count'] += 1
|
||
group['last_at'] = when
|
||
group['last_monotonic_us'] = monotonic
|
||
group['last_observed_at'] = observed
|
||
for row in groups.values():
|
||
emit(row.pop('category'), row.pop('severity'), **row)
|
||
state['recent_source_ids'] = recent[-4096:]
|
||
pending = rows >= 20000
|
||
if pending and not state.get('journal_backlog'):
|
||
emit('journal', 'WARN', detail='本轮读取 20000 条匹配记录；下轮从游标继续处理积压，未跳过中间记录。')
|
||
elif not pending and state.get('journal_backlog'):
|
||
emit('journal', 'INFO', detail='历史日志积压已处理完成，后续按游标增量采集。')
|
||
state['journal_backlog'] = pending
|
||
|
||
def pci_gpus():
|
||
items = []
|
||
for path in sorted(Path('/sys/bus/pci/devices').glob('*')):
|
||
if read(path / 'vendor') != '0x10de' or not read(path / 'class').startswith('0x03'):
|
||
continue
|
||
driver = path / 'driver'
|
||
items.append({'pci': path.name, 'device_id': read(path / 'device'),
|
||
'driver': driver.resolve().name if driver.exists() else None,
|
||
'link_speed': read(path / 'current_link_speed'), 'link_width': read(path / 'current_link_width')})
|
||
return items
|
||
|
||
def gpu_query():
|
||
args = ['nvidia-smi', '--query-gpu=pci.bus_id,name,uuid,temperature.gpu,power.draw,memory.total',
|
||
'--format=csv,noheader,nounits']
|
||
code, out, err = command(args)
|
||
result = []
|
||
if code == 0:
|
||
for row in csv.reader(out.splitlines(), skipinitialspace=True):
|
||
if len(row) != 6:
|
||
continue
|
||
try:
|
||
result.append({'pci': row[0], 'name': row[1], 'uuid': row[2],
|
||
'temperature_c': float(row[3]), 'power_w': float(row[4]), 'memory_mib': float(row[5])})
|
||
except ValueError:
|
||
result.append({'pci': row[0], 'name': row[1], 'uuid': row[2], 'raw': row})
|
||
return result, None if code == 0 and result else (err or out or 'nvidia-smi 未返回设备')[:1200]
|
||
|
||
def disk_names():
|
||
return sorted(p.name for p in Path('/sys/block').glob('*')
|
||
if not p.name.startswith(('loop', 'ram', 'zram', 'dm-', 'md')))
|
||
|
||
def sensors():
|
||
result = []
|
||
for path in sorted(Path('/sys/class/hwmon').glob('hwmon*')):
|
||
for inp in path.glob('temp*_input'):
|
||
try:
|
||
prefix = inp.name[:-6]
|
||
result.append({'chip': read(path / 'name'), 'label': read(path / (prefix + '_label'), prefix),
|
||
'celsius': int(read(inp)) / 1000,
|
||
'critical_c': int(read(path / (prefix + '_crit'), '0')) / 1000})
|
||
except ValueError:
|
||
pass
|
||
return result
|
||
|
||
def active_problem(problems, key, kind, detail, severity='ERROR'):
|
||
problems[key] = {'category': kind, 'severity': severity, 'detail': detail}
|
||
|
||
def smart_checks(state, problems, disks):
|
||
# Hourly, read-only SMART query. Never starts self-tests or changes disk settings.
|
||
cached = state.get('smart', {})
|
||
if time.time() - state.get('smart_at', 0) >= 3600:
|
||
cached = {}
|
||
for name in disks:
|
||
code, out, err = command(['smartctl', '-a', '-j', '/dev/' + name], 7)
|
||
try:
|
||
data = json.loads(out)
|
||
except ValueError:
|
||
cached[name] = {'available': False, 'detail': (err or out)[:500]}
|
||
continue
|
||
nv = data.get('nvme_smart_health_information_log', {})
|
||
cached[name] = {'available': 'smart_status' in data,
|
||
'passed': data.get('smart_status', {}).get('passed'),
|
||
'critical_warning': nv.get('critical_warning'), 'media_errors': nv.get('media_errors'),
|
||
'temperature_c': data.get('temperature', {}).get('current'),
|
||
'percentage_used': nv.get('percentage_used'), 'exit_status': code}
|
||
state['smart'], state['smart_at'] = cached, time.time()
|
||
for name, item in cached.items():
|
||
if not item.get('available'):
|
||
active_problem(problems, 'smart-unavailable:' + name, 'smart',
|
||
f'{name}: SMART 读取不可用，当前仅监控内核存储报错。{item.get("detail", "")}', 'WARN')
|
||
elif item.get('passed') is False or item.get('critical_warning', 0):
|
||
active_problem(problems, 'smart:' + name, 'smart', f'{name}: {json.dumps(item, ensure_ascii=False)}')
|
||
previous = state.get('smart_media', {}).get(name)
|
||
media = item.get('media_errors')
|
||
if media is not None and previous is not None and media > previous:
|
||
emit('smart', 'ERROR', detail=f'{name}: NVMe 介质错误计数从 {previous} 增加到 {media}')
|
||
if media is not None:
|
||
state.setdefault('smart_media', {})[name] = media
|
||
return cached
|
||
|
||
def collect(state, cfg):
|
||
now = stamp()
|
||
boot = read('/proc/sys/kernel/random/boot_id')
|
||
clock = {'wall': time.time(), 'monotonic': time.monotonic(), 'boot_id': boot}
|
||
old_clock = state.get('clock', {})
|
||
if old_clock.get('boot_id') == boot:
|
||
delta = (clock['wall'] - old_clock['wall']) - (clock['monotonic'] - old_clock['monotonic'])
|
||
if abs(delta) > 60:
|
||
emit('clock', 'WARN', detail=f'同一次启动中墙上时钟相对单调时钟变化 {delta:.1f} 秒；核对事件时间时应结合启动 ID 和单调时钟。')
|
||
state['clock'] = clock
|
||
if boot != state.get('boot_id'):
|
||
emit('boot', boot_id=boot, previous_boot_id=state.get('boot_id'),
|
||
detail='监控首次观察到本次启动；准确启动信息可通过 journalctl 查看。')
|
||
state['boot_id'] = boot
|
||
state['smart_at'] = 0
|
||
kernel_events(state, cfg)
|
||
pci = pci_gpus()
|
||
gpu, nv_error = gpu_query() if pci else ([], 'PCIe 总线没有 NVIDIA 显卡')
|
||
disks = disk_names()
|
||
temps = sensors()
|
||
problems = {}
|
||
if state.get('journal_error'):
|
||
active_problem(problems, 'journal-read', 'journal', state['journal_error'], 'WARN')
|
||
if len(pci) < cfg['expected_gpu_count']:
|
||
active_problem(problems, 'gpu-count', 'gpu_missing',
|
||
f'预期 {cfg["expected_gpu_count"]} 块 NVIDIA 显卡，PCIe 实际 {len(pci)} 块。')
|
||
if nv_error or len(gpu) < len(pci):
|
||
active_problem(problems, 'gpu-driver', 'gpu_driver', nv_error or f'PCIe {len(pci)} 块，nvidia-smi 仅 {len(gpu)} 块。')
|
||
if not nv_error:
|
||
missing = sorted(set(cfg.get('expected_gpu_uuids', [])) - {x['uuid'] for x in gpu})
|
||
if missing:
|
||
active_problem(problems, 'gpu-uuid', 'gpu_identity', '缺少预期 GPU：' + ', '.join(missing))
|
||
for item in gpu:
|
||
if item.get('temperature_c', 0) >= cfg.get('gpu_warn_c', 85):
|
||
active_problem(problems, 'gpu-temp:' + item['uuid'], 'gpu_temp',
|
||
f'{item["name"]} {item["pci"]}: {item["temperature_c"]}°C', 'WARN')
|
||
for name in set(cfg.get('expected_disks', [])) - set(disks):
|
||
active_problem(problems, 'disk:' + name, 'disk_missing', name)
|
||
for item in temps:
|
||
if item['critical_c'] > 0 and item['celsius'] >= item['critical_c']:
|
||
active_problem(problems, 'sensor:' + item['chip'] + item['label'], 'sensor_temp',
|
||
json.dumps(item, ensure_ascii=False), 'ERROR')
|
||
smart = smart_checks(state, problems, disks)
|
||
previous = state.get('active', {})
|
||
for key, value in problems.items():
|
||
old = previous.get(key)
|
||
value['since'] = old.get('since', now) if old else now
|
||
if not old or old.get('detail') != value['detail']:
|
||
emit(value['category'], value['severity'], event='FAULT', key=key, detail=value['detail'], since=value['since'])
|
||
for key, value in previous.items():
|
||
if key not in problems:
|
||
emit(value['category'], 'INFO', event='RECOVERED', key=key, detail='此前异常在本轮检查中已消失。', since=value.get('since'))
|
||
state['active'] = problems
|
||
status = {'checked_at': now, 'boot_id': boot, 'pci_gpus': pci, 'gpus': gpu,
|
||
'nvidia_error': nv_error, 'disks': disks, 'smart': smart, 'sensors': temps, 'active': problems,
|
||
'journal_last_success': state.get('journal_last_success'), 'journal_backlog': state.get('journal_backlog', False)}
|
||
save(BASE / 'status.json', status)
|
||
if time.time() - state.get('snapshot_at', 0) >= 900:
|
||
path = LOG / ('samples-' + dt.datetime.now().astimezone().strftime('%Y-%m-%d') + '.jsonl')
|
||
with path.open('a', encoding='utf-8') as f:
|
||
f.write(json.dumps(status, ensure_ascii=False) + '\n')
|
||
os.chmod(path, 0o640)
|
||
state['snapshot_at'] = time.time()
|
||
save(BASE / 'state.json', state)
|
||
# Age and total-size limits apply only to this monitor's own log files.
|
||
files = sorted(LOG.glob('*.jsonl'), key=lambda p: p.stat().st_mtime)
|
||
cutoff = time.time() - cfg.get('retention_days', 90) * 86400
|
||
total = sum(p.stat().st_size for p in files)
|
||
for path in files:
|
||
if path.stat().st_mtime < cutoff or total > 256 * 1024 * 1024:
|
||
if path.name.endswith(dt.datetime.now().strftime('%Y-%m-%d') + '.jsonl'):
|
||
continue
|
||
total -= path.stat().st_size
|
||
path.unlink()
|
||
|
||
def show_status():
|
||
status = load(BASE / 'status.json', {})
|
||
print('硬件管理日志 | 最近检查：' + status.get('checked_at', '尚未运行'))
|
||
age = (dt.datetime.now().astimezone() - dt.datetime.fromisoformat(status['checked_at'])).total_seconds() if status else 9999
|
||
if age > 180:
|
||
print('警告：最近检查超过 3 分钟，请检查 systemctl status hardware-watch.timer hardware-watch.service')
|
||
for item in status.get('pci_gpus', []):
|
||
print(f'PCIe {item["pci"]} | {item["link_speed"]} x{item["link_width"]} | {item["driver"]}')
|
||
for item in status.get('gpus', []):
|
||
print(f'{item["name"]} | {item["pci"]} | {item.get("temperature_c", "?")}°C | {item.get("power_w", "?")}W | {item["uuid"]}')
|
||
for name, item in status.get('smart', {}).items():
|
||
print(f'磁盘 {name} | SMART={item.get("passed")} | 温度={item.get("temperature_c")}°C | 介质错误={item.get("media_errors")}')
|
||
print('当前采样告警：' + str(len(status.get('active', {}))))
|
||
for value in status.get('active', {}).values():
|
||
print(f' [{value["severity"]}] {NAMES[value["category"]]}：{value["detail"]}')
|
||
if status.get('journal_backlog'):
|
||
print('历史内核日志正在补录，实时 GPU／磁盘采样已启用。')
|
||
print('PCIe 等内核历史事件另见 hwlog events；当前在线不代表此前没有通信错误。')
|
||
|
||
def show_events(args):
|
||
rows = []
|
||
seen = set()
|
||
status = load(BASE / 'status.json', {})
|
||
active = status.get('active', {})
|
||
since = dt.datetime.fromisoformat(args.since).astimezone() if args.since else None
|
||
for path in sorted(LOG.glob('events-*.jsonl')):
|
||
for line in path.read_text().splitlines():
|
||
try:
|
||
row = json.loads(line)
|
||
if row.get('source') == 'kernel':
|
||
key = (row.get('boot_id'), row.get('category'), row.get('device'),
|
||
row.get('first_monotonic_us'), row.get('last_monotonic_us'), row.get('detail'))
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
event_time = row.get('first_at', row['observed_at'])
|
||
source_last = row.get('last_at', event_time)
|
||
observed = row.get('observed_at', event_time)
|
||
if since and max(source_last, observed) < since.isoformat():
|
||
continue
|
||
if args.category and row['category'] != args.category:
|
||
continue
|
||
rows.append(row)
|
||
except (ValueError, KeyError):
|
||
continue
|
||
# Recently collected records remain visible even if old source clocks were ahead.
|
||
rows.sort(key=lambda x: x['observed_at'])
|
||
for row in rows[-max(1, args.limit):]:
|
||
if args.json:
|
||
print(json.dumps(row, ensure_ascii=False))
|
||
else:
|
||
print(f'{row.get("first_at", row["observed_at"])} [{row["severity"]}] {row["summary"]} {row.get("device", "")} {row.get("event", "")}')
|
||
if 'count' in row:
|
||
print(f' 原始故障时间：{row.get("first_at", "未知")} 至 {row.get("last_at", "未知")}')
|
||
print(f' 监控发现时间：{row.get("first_observed_at", row["observed_at"])} 至 {row.get("last_observed_at", row["observed_at"])}')
|
||
first_mono = row.get('first_monotonic_us')
|
||
last_mono = row.get('last_monotonic_us')
|
||
if first_mono and last_mono:
|
||
print(f' 本次启动经过：{int(first_mono)/1000000:.3f}s 至 {int(last_mono)/1000000:.3f}s；启动 {row.get("boot_id", "")}')
|
||
print(f' 本批重复次数：{row["count"]}')
|
||
current = False
|
||
for value in active.values():
|
||
detail = value.get('detail', '')
|
||
if value.get('category') == row.get('category') and (not row.get('device') or row.get('device') in detail):
|
||
current = True
|
||
break
|
||
print(' 当前判断：当前采样仍有同类告警' if current else ' 当前判断：当前采样未复现，属于历史记录或已恢复')
|
||
if row['last_at'] < row['first_at']:
|
||
print(' 时间提示：源日志时钟发生回拨，以上首末时间不能按日历顺序判断；请结合启动 ID、单调时钟及原始 journal。')
|
||
if row['first_at'] > row['observed_at']:
|
||
print(' 时间提示：源日志日期晚于采集日期，源系统时钟曾异常；采集时间为 ' + row['observed_at'])
|
||
print(' ' + row.get('detail', ''))
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description='硬件管理日志：每分钟采样，查看状态和历史故障。')
|
||
sub = parser.add_subparsers(dest='action')
|
||
sub.add_parser('collect')
|
||
sub.add_parser('status')
|
||
events = sub.add_parser('events')
|
||
events.add_argument('--since', help='例如 2026-11-02 或 2026-11-02T12:00:00+08:00')
|
||
events.add_argument('--category', choices=sorted(NAMES))
|
||
events.add_argument('--limit', type=int, default=50)
|
||
events.add_argument('--json', action='store_true')
|
||
sub.add_parser('self-test')
|
||
args = parser.parse_args()
|
||
if args.action == 'self-test':
|
||
cases = [('NVRM: GPU does not have the necessary power cables connected.', 'power'),
|
||
('NVRM: RmInitAdapter failed! (0x24:0x1c:1603)', 'gpu_init'),
|
||
('PCIe Bus Error: severity=Correctable, type=Data Link Layer', 'pcie_correctable'),
|
||
('PCIe Bus Error: severity=Uncorrectable (Fatal)', 'pcie_fatal'),
|
||
('NVRM: Xid (PCI:0000:03:00): 79, GPU has fallen off the bus.', 'gpu_xid'),
|
||
('nvme nvme0: I/O 10 timeout, reset controller', 'storage'),
|
||
('EDAC MC0: 1 CE memory read error', 'memory_cpu')]
|
||
for message, expected in cases:
|
||
assert classify(message)[0] == expected, message
|
||
assert classify('NVRM: loading NVIDIA UNIX x86_64 Kernel Module') is None
|
||
assert classify('nvidia: module verification failed: tainting kernel') is None
|
||
print('PASS: 7 hardware error cases and 2 benign driver messages')
|
||
return
|
||
if args.action == 'collect':
|
||
BASE.mkdir(parents=True, exist_ok=True)
|
||
LOG.mkdir(parents=True, exist_ok=True)
|
||
with (BASE / 'lock').open('w') as lock:
|
||
try:
|
||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
except BlockingIOError:
|
||
return
|
||
collect(load(BASE / 'state.json', {}), load(CONFIG, {}))
|
||
elif args.action == 'events':
|
||
show_events(args)
|
||
else:
|
||
show_status()
|
||
|
||
if __name__ == '__main__':
|
||
main()
|