jp/asterisk-voicemail
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MySQL backend + CDR + contacts email + matcher fix

Browse Source 
- vm_store: MySQL (PyMySQL) default backend, SQLite fallback
- CDR: cdr_adaptive_odbc + res_odbc + MariaDB ODBC driver + Master.csv backfill
- contacts: full E.164 matching, WORK-email preference, junk-name guard
- backfill scripts: vm_migrate_sqlite_to_mysql.py, vm_backfill_contacts.py,
  vm_backfill_cdr.py
- live: 131 messages migrated, 7591 CDR rows backfilled
This commit is contained in:
jp
2026-08-13 13:09:03 +01:00
parent 21dc883e99
commit 05c5dbe45a
6 changed files with 531 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vm_contacts.py
+81 -37
View File
@@ -34,13 +34,30 @@ def digits_of(s):
return re.sub(r"\D", "", s or "")
def normalize_uk(d):
"""Canonicalise a UK number so +44 and 0 forms collide.
Asterisk stores caller IDs as '+441273003930' while a Google export holds
'01273003930' (and vice-versa). Both are the same number; strip the leading
'44' (from +44) and prepend '0', and turn a '00' intl prefix into '00'.
Non-UK numbers are returned unchanged.
"""
if not d:
return d
if d.startswith("44") and len(d) == 12: # +441273003930
return "0" + d[2:]
if d.startswith("0044") and len(d) == 14: # 00441273003930
return "0" + d[4:]
return d
def extract_number(caller):
"""Pull a dialable number out of a CallerID string like 'Dave <079...>'."""
if not caller:
return ""
m = re.search(r"<([^>]+)>", caller)
cand = m.group(1) if m else caller
return digits_of(cand)
return normalize_uk(digits_of(cand))
def _key(num_digits, n):
@@ -82,8 +99,9 @@ class Cache:
return None
return e # {"t":..., "name": <str or None>}
def put(self, key, name):
self.data[key] = {"t": time.time(), "name": name}
def put(self, key, result):
# result is the (name, email) tuple from resolve().
self.data[key] = {"t": time.time(), "name": result[0], "email": result[1]}
try:
os.makedirs(os.path.dirname(self.path), exist_ok=True)
tmp = self.path + ".tmp"
@@ -98,20 +116,30 @@ class Cache:
def _parse_vcf(text):
"""Return list of (name, [numbers])."""
out = []
name, nums = None, []
name, nums, emails = None, [], []
for raw in text.splitlines():
line = raw.strip()
u = line.upper()
if u == "BEGIN:VCARD":
name, nums = None, []
name, nums, emails = None, [], []
elif u.startswith("FN"):
name = line.split(":", 1)[1].strip() if ":" in line else None
elif u.startswith("TEL"):
if ":" in line:
nums.append(line.split(":", 1)[1].strip())
elif u.startswith("EMAIL"):
if ":" in line:
# Prefer a WORK-typed address when one exists; callers usually
# want the business reply address, not a personal/home one.
is_work = "WORK" in line.upper()
addr = line.split(":", 1)[1].strip()
if is_work:
emails.insert(0, addr) # work goes first
else:
emails.append(addr)
elif u == "END:VCARD":
if name and nums:
out.append((name, nums))
out.append((name, nums, emails))
return out
@@ -124,6 +152,8 @@ def _parse_csv(path):
name_cols = [c for c in cols if c and ("Name" == c or c.endswith("Name"))]
phone_cols = [c for c in cols if c and "Phone" in c and "Value" in c] or \
[c for c in cols if c and "Phone" in c]
email_cols = [c for c in cols if c and "E-mail" in c] or \
[c for c in cols if c and "Email" in c]
for row in r:
name = ""
if "Name" in row and row["Name"]:
@@ -135,27 +165,25 @@ def _parse_csv(path):
for c in phone_cols:
v = row.get(c, "")
if v:
nums.extend(re.split(r"\s*:::\s*|\s*;\s*", v))
nums.extend(re.split(r"\s*:::\s*|\s*;", v))
emails = []
for c in email_cols:
v = row.get(c, "")
if v:
emails.extend(re.split(r"\s*:::\s*|\s*;", v))
if name and nums:
out.append((name.strip(), nums))
out.append((name.strip(), nums, emails))
return out
def _index(entries, n):
"""Index contact numbers by their full digit string (capped at n, default
15 = max E.164 length), NOT just the trailing 9 digits.
"""Index contact numbers -> (name, email) by full digit string.
Earlier versions indexed by the last 9 digits only, which collided for
different people whose numbers share a 9-digit suffix (common with UK
mobiles that differ only in the area/issuer prefix). Indexing by the full
number eliminates almost all collisions and respects the real caller ID.
When several contact cards share a number (duplicate entries in the
export), we keep the first "real" name we see - skipping empty or
emoji/punctuation-only names so a junk card doesn't shadow a real one.
See resolve()/lookup_file() for the collision rules. Returns a dict
keyed by the last n digits of each number, value (name, email_or_None).
"""
idx = {}
for name, nums in entries:
for name, nums, emails in entries:
if not name or not nums:
continue
if _is_junk_name(name):
@@ -164,7 +192,8 @@ def _index(entries, n):
if not d:
continue
k = d[-n:] if len(d) >= n else d
idx.setdefault(k, name)
if k not in idx:
idx[k] = (name, (emails[0] if emails else None))
return idx
@@ -179,15 +208,13 @@ def lookup_file(cfg, num_digits, n, log):
with open(path, encoding="utf-8", errors="replace") as fh:
entries = _parse_vcf(fh.read())
index = _index(entries, n)
# Try the longest available match first (full digits), then progressively
# shorter tails, so a full-number hit wins over a 9-digit tail collision.
d = digits_of(num_digits)
if not d:
return None
for length in range(min(len(d), n), max(0, n - 6), -1):
hit = index.get(d[-length:])
if hit:
return hit
return hit # (name, email)
return None
except Exception as e:
log("contacts file backend error: %s" % e)
@@ -275,7 +302,10 @@ def lookup_google(cfg, num_digits, n, log):
if _key(ph.get("value", ""), n) == want:
names = person.get("names", [])
if names:
return names[0].get("displayName")
emails = [e.get("value") for e in person.get("emailAddresses", [])
if e.get("value")]
return (names[0].get("displayName"),
(emails[0] if emails else None))
except Exception as e:
log("google People API error: %s" % e)
return None
@@ -319,7 +349,7 @@ def lookup_carddav(cfg, num_digits, n, log):
entries = []
for v in vcards:
entries.extend(_parse_vcf(v.replace("&#13;", "").replace("\r", "")))
return _index(entries, n).get(_key(num_digits, n))
return _index(entries, n).get(_key(num_digits, n)) # (name, email)
except Exception as e:
log("carddav backend error: %s" % e)
return None
@@ -330,27 +360,35 @@ _BACKENDS = {"file": lookup_file, "google": lookup_google, "carddav": lookup_car
def resolve(caller_id, log=print):
"""Return a contact name for a CallerID string, or None."""
"""Return (name, email) for a CallerID string, or (None, None).
`name` is the resolved contact name (or None); `email` is the contact's
email when the matched number also has one in the address book (or None).
"""
num = extract_number(caller_id)
if not num or not os.path.exists(CONF_PATH):
return None
return (None, None)
try:
cp = configparser.ConfigParser(inline_comment_prefixes=("#", ";"))
cp.read(CONF_PATH)
if not cp.has_section("contacts") or not cp["contacts"].getboolean("enabled", False):
return None
return (None, None)
g = cp["contacts"]
n = int(g.get("match_digits", "9") or 9)
n = int(g.get("match_digits", "15") or 15)
cache = Cache(g.get("cache_path", "/var/lib/vm-transcribe/contacts_cache.json"),
int(g.get("cache_ttl", "86400") or 86400))
key = _key(num, n)
cached = cache.get(key)
if cached is not None:
return cached.get("name")
# The cache stores {"t":..., "name": name_str, "email": email_str}.
if isinstance(cached, dict):
return (cached.get("name") or None, cached.get("email") or None)
# legacy: stored the name directly (no dict wrapper)
return (cached, None)
order = [b.strip() for b in g.get("backends", "file").split(",") if b.strip()]
name = None
result = (None, None)
for b in order:
fn = _BACKENDS.get(b)
if not fn:
@@ -358,19 +396,25 @@ def resolve(caller_id, log=print):
continue
sect = cp[b] if cp.has_section(b) else {}
try:
name = fn(sect, num, n, log)
hit = fn(sect, num, n, log)
except Exception as e:
log("contacts backend %s crashed: %s" % (b, e))
name = None
hit = None
if isinstance(hit, tuple):
name, email = hit
else:
name, email = hit, None # legacy backends returned name only
if name:
log("contacts: %s -> %s (via %s)" % (num, name, b))
result = (name, email)
log("contacts: %s -> %s (via %s)%s"
% (num, name, b, (" email=%s" % email if email else "")))
break
cache.put(key, name) # cache misses too (name=None)
return name
cache.put(key, result) # cache the (name, email) tuple
return result
except Exception as e:
log("contacts resolve error: %s" % e)
return None
return (None, None)
if __name__ == "__main__":