Clear[M, a1, b1, a2, b2, a3, b3, a4, b4];
M = {{a1, a2, a3, a4}, {b1, b2, b3, b4}};
Do[If[Mod[Transpose[M].( { {0, 1}, {-1, 0} } ).M, 6] == Mod[ConstantArray[0, {4, 4}], 6] &&
Mod[a1, 6] != Mod[0, 6] &&
Mod[a2 - b2, 6] != Mod[0, 6] != Mod[2*a3 - 5*b3, 6] != Mod[0, 6] &&
Mod[b4, 6] != Mod[0, 6], Print[M]], {a1, 0, 5}, {b1, 0, 5}, {a2,
0, 5}, {b2, 0, 5}, {a3, 0, 5}, {b3, 0, 5}, {a4, 0, 5}, {b4, 0, 5}]

Mod[a2 - b2, 6] != Mod[0, 6] != Mod[2*a3 - 5*b3, 6] != Mod[0, 6]这句摆明了永不成立。你是不是把 && 错成 != 了。