jp
05c5dbe45a
- vm_store: MySQL (PyMySQL) default backend, SQLite fallback - CDR: cdr_adaptive_odbc + res_odbc + MariaDB ODBC driver + Master.csv backfill - contacts: full E.164 matching, WORK-email preference, junk-name guard - backfill scripts: vm_migrate_sqlite_to_mysql.py, vm_backfill_contacts.py, vm_backfill_cdr.py - live: 131 messages migrated, 7591 CDR rows backfilled
99 lines
3.2 KiB
Python
99 lines
3.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Backfill Asterisk Master.csv CDR rows into the MySQL `cdr` table.
|
|
|
|
CSV column mapping for Asterisk 16 default cdr_csv.conf:
|
|
calldate=9, clid=4, src=1, dst=2, dcontext=3,
|
|
channel=5, dstchannel=6, lastapp=7, lastdata=8,
|
|
duration=12, billsec=13, disposition=14, amaflags=15,
|
|
accountcode=0, uniqueid=16, userfield=17
|
|
peeraccount/linkedid/sequence are not in the CSV -> NULL
|
|
|
|
Idempotent: skips rows whose uniqueid already exists; for rows with no
|
|
uniqueid (early Asterisk) dedups on a row-content hash.
|
|
|
|
Run as asterisk:
|
|
sudo -u asterisk /opt/vm-transcribe/venv/bin/python3 /opt/vm-transcribe/vm_backfill_cdr.py
|
|
"""
|
|
import csv
|
|
import hashlib
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import vm_store
|
|
|
|
CSV_PATH = "/var/log/asterisk/cdr-csv/Master.csv"
|
|
# CSV positions (0-based) as parsed from a real Master.csv row
|
|
POS = {
|
|
"accountcode": 0, "src": 1, "dst": 2, "dcontext": 3, "clid": 4,
|
|
"channel": 5, "dstchannel": 6, "lastapp": 7, "lastdata": 8,
|
|
"calldate": 9, "duration": 12, "billsec": 13, "disposition": 14,
|
|
"amaflags": 15, "uniqueid": 16, "userfield": 17,
|
|
}
|
|
# Column order matching the MySQL cdr table
|
|
COLS = [
|
|
"calldate", "clid", "src", "dst", "dcontext",
|
|
"channel", "dstchannel", "lastapp", "lastdata",
|
|
"duration", "billsec", "disposition", "amaflags",
|
|
"accountcode", "uniqueid", "peeraccount", "linkedid",
|
|
"sequence", "userfield",
|
|
]
|
|
# Which of those are NULL because not in the CSV
|
|
NULL_AFTER = {"peeraccount", "linkedid", "sequence"}
|
|
|
|
|
|
def parse(v):
|
|
if v is None or v == "":
|
|
return None
|
|
try:
|
|
return int(v)
|
|
except ValueError:
|
|
return v
|
|
|
|
|
|
def main():
|
|
if not os.path.exists(CSV_PATH):
|
|
print("csv not found:", CSV_PATH)
|
|
return
|
|
con = vm_store.connect()
|
|
seen = set()
|
|
total = 0
|
|
skip = 0
|
|
with open(CSV_PATH, newline="", encoding="utf-8", errors="replace") as f:
|
|
reader = csv.reader(f, quotechar='"', escapechar="\\", doublequote=False)
|
|
for row in reader:
|
|
if not row or len(row) < 17:
|
|
continue
|
|
uid = (row[POS["uniqueid"]] if len(row) > POS["uniqueid"] else "") or ""
|
|
key = uid
|
|
if not key:
|
|
key = hashlib.sha1(("\x00".join(row)).encode("utf-8", errors="replace")).hexdigest()[:32]
|
|
if key in seen:
|
|
skip += 1
|
|
continue
|
|
seen.add(key)
|
|
total += 1
|
|
vals = []
|
|
for c in COLS:
|
|
if c in NULL_AFTER:
|
|
vals.append(None)
|
|
continue
|
|
v = row[POS[c]] if POS[c] < len(row) else None
|
|
vals.append(parse(v))
|
|
ph = ",".join(["%s"] * len(COLS))
|
|
try:
|
|
con.execute(
|
|
"INSERT IGNORE INTO cdr (%s) VALUES (%s)"
|
|
% (",".join(COLS), ph),
|
|
vals,
|
|
)
|
|
except Exception as e:
|
|
print(" skip key=%s: %s" % (key, e))
|
|
con.commit()
|
|
mysql_total = con.execute("SELECT COUNT(*) c FROM cdr").fetchone()["c"]
|
|
print("csv scanned: %d skipped: %d -> MySQL cdr total: %d" % (total, skip, mysql_total))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|