ME=MatrixExp;

ρt=(({{1, 0}, {0, 1}}));  ρy=(({{0, 0}, {-ϵ, 0}}));  ρa=(({{(1+1/ϵ)/2, 0}, {0, -(1-1/ϵ)/2}})); ρx=(({{0, 1}, {0, 0}}));
Simplify@{ρa.ρx-ρx.ρa==ρx, ρa.ρy-ρy.ρa==-ρy, ρx.ρy-ρy.ρx==ρt-2ϵ ρa}

(prod=ME[τ ρt].ME[η ρy].ME[α ρa].ME[ξ ρx]) // MatrixForm

{tr=Tr@prod, det=Det@prod} // Simplify

{tr,det}/.ϵ->0

tr/(Sqrt[det]) //PowerExpand// FullSimplify

tr/(Sqrt[det]) //PowerExpand// FullSimplify // Expand

Tr[ME[η ρy].ME[λ(ρt-2ϵ ρa)].ME[ξ ρx]]//Simplify