4 hours ago
(4 hours ago)nablator wrote: You are not allowed to view links. Register or Login to view.About parameters... what can be improved?
I think trying the transition matrix instead of TFID, would be the main difference.
In python you could do:
Code:
from collections import Counter
import numpy as np
from scipy.spatial.distance import pdist, squareform
# Assumes spaces separate words. Adapt if the transcription uses dots.
def transitions(text):
counts = Counter()
for word in text.split():
units = ["<START>", *word, "<END>"]
counts.update(zip(units, units[1:]))
return counts
profiles = [transitions(text) for text in df["text"]]
features = sorted(set().union(*(p.keys() for p in profiles)))
# Each row is a flattened character-to-character transition matrix.
X = np.array([
[p.get(pair, 0) for pair in features]
for p in profiles
], dtype=float)
if np.any(X.sum(axis=1) == 0):
raise ValueError("Remove empty texts before fitting.")
X /= X.sum(axis=1, keepdims=True)
# Already the square root of JS divergence—do not sqrt again.
distance_matrix = squareform(pdist(X, metric="jensenshannon"))i think that should be able to be fed into distance_matrix in your current MDS call. The only other thing would be designing your glyph set.
Code:
# Illustrative only: choose units appropriate to their transcription.
GLYPHS = [
"ckh", "cth", "cph", "cfh",
"ch", "sh",
"a", "o", "e", "i", "y",
"k", "t", "p", "f",
"q", "d", "s", "r", "l", "n", "m",
]
# Match longer glyph spellings before shorter ones.
GLYPHS = sorted(set(GLYPHS), key=lambda g: (-len(g), g))
def tokenize(word):
result = []
pos = 0
while pos < len(word):
for glyph in GLYPHS:
if word.startswith(glyph, pos):
result.append(glyph)
pos += len(glyph)
break
else:
raise ValueError(f"Unknown glyph in {word!r} at position {pos}")
return result
def transitions(text):
counts = Counter()
for word in text.split():
units = ["<START>", *tokenize(word), "<END>"]
counts.update(zip(units, units[1:]))
return counts